Created
July 25, 2022 03:25
-
-
Save darothen/5380e223ae5bc894006a5b6ed5a27cbb to your computer and use it in GitHub Desktop.
Strawman for GRIB2 pangeo-forge recipe
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from pangeo_forge_recipes.patterns import pattern_from_file_sequence, FilePattern, ConcatDim | |
| from pangeo_forge_recipes.recipes import HDFReferenceRecipe | |
| from pathlib import Path | |
| from dask.diagnostics import ProgressBar | |
| # NOTE(darothen): use PR fsspec/kerchunk#204 | |
| # from kerchunk.grib2 import scan_grib | |
| import intake | |
| ## Local | |
| # NOTE(darothen): Download some recent HRRR model output to a local directory, or | |
| # use the remote block (commented out) below to access it from S3. Note that you may | |
| # need to modify `storage_options` if accessing the remote data. | |
| # The following curl command can help automate downloading the HRRR data, modify it for | |
| # a more recent YYYYMMDD (only two days are archived on NOMADS): | |
| # $ curl -O "https://nomads.ncep.noaa.gov/pub/data/nccf/com/hrrr/prod/hrrr.20220721/conus/hrrr.t00z.wrfsfcf[00-18].grib2" | |
| data_path = Path("./hrrr_data/") | |
| fn_pths = ["file://"+str(pth) for pth in sorted(data_path.glob("*.grib2"))] | |
| fn_pths = fn_pths[:5] | |
| pattern = pattern_from_file_sequence(fn_pths, 'step', file_type='grib') | |
| ## Remote | |
| # _tpl = "s3://noaa-hrrr-bdp-pds/hrrr.20220722/conus/hrrr.t22z.wrfsfcf{step:02d}.grib2" | |
| # _tpl = "https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20220722/conus/hrrr.t22z.wrfsfcf{step:02d}.grib2" | |
| # def format_function(step: int) -> str: | |
| # return _tpl.format(step=step) | |
| # pattern = FilePattern( | |
| # format_function, ConcatDim("step", [0, 1, 2, 3]), | |
| # file_type='grib', | |
| # ) | |
| storage_options = { | |
| 'anon': True, | |
| 'default_cache_type': 'readahead', | |
| } | |
| _common = ['time', 'step', 'latitude', 'longitude', 'valid_time'] | |
| storage_options['common_coords'] = _common | |
| _filter = { | |
| ## Option 1) Grab all the variables defined on the 850 mb surface | |
| # 'typeOfLevel': 'isobaricInhPa', | |
| # 'level': 850, | |
| # 'shortName': 't' | |
| ## Option 2) Down-select solely to surface (skin) temperature | |
| 'typeOfLevel': 'surface', | |
| 'shortName': 't' | |
| } | |
| storage_options['filter_by_keys'] = _filter | |
| recipe = HDFReferenceRecipe(pattern, src_storage_options=storage_options) | |
| # recipe = recipe.copy_pruned() | |
| # recipe_func = recipe.to_function() | |
| # recipe_func() | |
| delayed = recipe.to_dask() | |
| with ProgressBar(): | |
| delayed.compute() | |
| cat_url = f"{recipe.target}/reference.yaml" | |
| cat = intake.open_catalog(cat_url) | |
| cat | |
| ds = cat.data.to_dask() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment