update to the official new CDS (#44)

yunjunz · yuankailiu · web-flow · commit cce7fa6bc224 · 2025-02-08T10:40:43.000+08:00
+ `autoget.py`:
   - update `cds_url` link
   - update to the latest syntax of `cdsapi` for the request dictionary input
   - use `f'{}'` instead of `.format` syntax

+ `README.md`: update account setup doc for the official new CDS

+ `requirements.txt`: the new CDS requires cdsapi&gt;=0.7.2

+ update `model.cfg` and `tests/test_dload.py` for the new CDS

+ `processor.intP2H()`: add whitespace for better readability

---------

Co-authored-by: Yuan-Kai Liu &lt;55262505+yuankailiu@users.noreply.github.com&gt;
diff --git a/README.md b/README.md
@@ -59,19 +59,19 @@ python PyAPS/tests/test_calc.py
 
 ### 2. Account setup for [ERA5](https://www.ecmwf.int/en/forecasts/dataset/ecmwf-reanalysis-v5)
 
-ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS)-beta ([migration guide](https://confluence.ecmwf.int/display/CKB/Please+read%3A+CDS+and+ADS+migrating+to+new+infrastructure%3A+Common+Data+Store+%28CDS%29+Engine)). Registration is required for the data access and downloading.
+ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS). Registration is required for the data access and downloading.
 
-+ [Create a new account](https://cds-beta.climate.copernicus.eu/) on the CDS-beta website if you don't own a user account yet. Note: the old CDS account won't work.
-+ [CDS API setup](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client): Create the local file `$HOME/.cdsapirc` (in your Unix/Linux environment) and add the following two lines:
++ [Create a new account](https://cds.climate.copernicus.eu/) on the CDS website if you don't own a user account yet. Note: the old CDS account won't work ([Goodbye legacy CDS, Hellow New CDS!](https://forum.ecmwf.int/t/goodbye-legacy-climate-data-store-hello-new-climate-data-store-cds/6380)).
++ [CDS API setup](https://cds.climate.copernicus.eu/how-to-api): Create the local file `$HOME/.cdsapirc` (in your Unix/Linux environment) and add the following two lines:
 
 ```shell
-url: https://cds-beta.climate.copernicus.eu/api
+url: https://cds.climate.copernicus.eu/api
 key: your-personal-access-token
 ```
 
-Your Personal Access Token can be found under [Your profile > Personal Access Token](https://cds-beta.climate.copernicus.eu/profile) section or on the [setup guide](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client) page. Alternatively, you could add the token to the `[CDS]` section in `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda. Note: using your [old CDS API key](https://cds.climate.copernicus.eu/) will lead to a 401 Client Error and Authentication failed.
+Your Personal Access Token can be found under [Your profile > Personal Access Token](https://cds.climate.copernicus.eu/profile) section or on the [setup guide](https://cds.climate.copernicus.eu/how-to-api) page. Alternatively, you could add the token to the `[CDS]` section in `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda. Note: using your legacy CDS API key will lead to a 401 Client Error and Authentication failed.
 
-+ **Make sure** that you accept the data license in the Terms of use on ECMWF website: Login, under [Datasets > ERA5 hourly data on pressure levels from 1940 to present > Download > Terms of use](https://cds-beta.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels?tab=download), click **Accept** to accespt the license to use Copernicus Products.
++ **Make sure** that you accept the data license in the Terms of use on ECMWF website: Login, under [Datasets > ERA5 hourly data on pressure levels from 1940 to present > Download > Terms of use](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels?tab=download), click **Accept** to accespt the license to use Copernicus Products.
 
 + Test the account setup by running:
 
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-cdsapi>=0.7.0
+cdsapi>=0.7.2
 numpy
 pygrib
 scipy
diff --git a/src/pyaps3/autoget.py b/src/pyaps3/autoget.py
@@ -41,32 +41,26 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non
     # Initialize
 
     # Check data
-    assert model in ('ERA5', 'ERAINT', 'HRES'), 'Unknown model for ECMWF: {}'.format(model)
+    assert model in ('ERA5', 'ERAINT', 'HRES'), f'Unknown model for ECMWF: {model}'
 
     # Infos for downloading
-    if model in 'ERAINT':
-        print('WARNING: you are downloading from the old ECMWF platform. '
+    if model == 'ERAINT':
+        print('WARNING: you are downloading from the legacy ECMWF platform. '
               'ERA-Interim is deprecated, use ERA-5 instead.')
-    if model in 'ERA5':
-        cds_url = 'https://cds-beta.climate.copernicus.eu/api'
+    if model == 'ERA5':
+        cds_url = 'https://cds.climate.copernicus.eu/api'
         print('INFO: You are using the latest ECMWF platform for downloading datasets: ', cds_url)
 
     #-------------------------------------------
     # Define parameters
 
     # Humidity
     assert humidity in ('Q','R'), 'Unknown humidity field for ECMWF'
-    if humidity in 'Q':
+    if humidity == 'Q':
         humidparam = 'specific_humidity' if model == 'ERA5' else 133
-    elif humidity in 'R':
+    elif humidity == 'R':
         humidparam = 'relative_humidity' if model == 'ERA5' else 157
 
-    # Grid size (only for HRES and ERA-Interim)
-    if model in 'HRES':
-        gridsize = '0.10/0.10'
-    elif model in 'ERAINT':
-        gridsize = '0.75/0.75'
-
     #-------------------------------------------
     # file name
     if not flist:
@@ -92,7 +86,7 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non
 
         #-------------------------------------------
         # CASE 1: request for CDS API client (new ECMWF platform, for ERA5)
-        if model in 'ERA5':
+        if model == 'ERA5':
             # Contact the server
             rc_file = os.path.expanduser('~/.cdsapirc')
             if os.path.isfile(rc_file):
@@ -102,39 +96,49 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non
                 key = config.get('CDS', 'key')
                 c = cdsapi.Client(url=url, key=key)
 
-            # Pressure levels
-            pressure_lvls = ['1','2','3','5','7','10','20','30','50',
-                            '70','100','125','150','175','200','225',
-                            '250','300','350','400','450','500','550',
-                            '600','650','700','750','775','800','825',
-                            '850','875','900','925','950','975','1000']
-
-            # Dictionary
-            indict = {'product_type'   :'reanalysis',
-                      'format'         :'grib',
-                      'variable'       :['geopotential','temperature','{}'.format(humidparam)],
-                      'pressure_level' : pressure_lvls,
-                      'year'           :'{}'.format(day[0:4]),
-                      'month'          :'{}'.format(day[4:6]),
-                      'day'            :'{}'.format(day[6:8]),
-                      'time'           :'{}:00'.format(hr)}
+            # request dictionary
+            pressure_lvls = [
+                '1','2','3','5','7','10','20','30','50',
+                '70','100','125','150','175','200','225',
+                '250','300','350','400','450','500','550',
+                '600','650','700','750','775','800','825',
+                '850','875','900','925','950','975','1000',
+            ]
+
+            indict = {
+                'product_type'   : ['reanalysis'],
+                'variable'       : ['geopotential', 'temperature', f'{humidparam}'],
+                'year'           : [f'{day[0:4]}'],
+                'month'          : [f'{day[4:6]}'],
+                'day'            : [f'{day[6:8]}'],
+                'time'           : [f'{hr}:00'],
+                'pressure_level' : pressure_lvls,
+                'data_format'    : 'grib',
+            }
 
             # download a geographical area subset
             if snwe is not None:
                 s, n, w, e = snwe
-                indict['area'] = '/'.join(['{:.2f}'.format(x) for x in [n, w, s, e]])
+                indict['area'] = [n, w, s, e]
 
             # Assert grib file not yet downloaded
             if not os.path.exists(fname):
-                print('Downloading %d of %d: %s '%(i+1, len(bdate), fname))
+                print(f'Downloading {i+1} of {len(bdate)}: {fname} ')
                 print(indict)
 
                 # Make the request
-                c.retrieve('reanalysis-{}-pressure-levels'.format(model.lower()),indict,target=fname)
+                ds_name = f'reanalysis-{model.lower()}-pressure-levels'
+                c.retrieve(ds_name, indict, target=fname)
 
         #-------------------------------------------
         # CASE 2: request for WEB API client (old ECMWF platform, deprecated, for ERA-Int and HRES)
         else:
+            # Grid size (only for HRES and ERA-Interim)
+            if model == 'HRES':
+                gridsize = '0.10/0.10'
+            elif model == 'ERAINT':
+                gridsize = '0.75/0.75'
+
             # Contact the server
             from pyaps3.ecmwfapi import ECMWFDataServer
             url = "https://api.ecmwf.int/v1"
diff --git a/src/pyaps3/model.cfg b/src/pyaps3/model.cfg
@@ -1,6 +1,6 @@
 # vim: set filetype=cfg:
-#####key for ECMWF in Climate Data Store Application Program Interface
-#Get it from https://retostauffer.org/code/Download-ERA5/
+#####Personal Access Token for ECMWF in Climate Data Store Application Program Interface
+#Get it from https://cds.climate.copernicus.eu/profile
 #for ERA5
 [CDS]
 key = your-personal-access-token
diff --git a/src/pyaps3/processor.py b/src/pyaps3/processor.py
@@ -111,12 +111,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False):
             #Interpolating pressure values
             hy = lvls.copy()
             if (sFlag == True):
-                val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3])
+                val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate((hy,[val]),axis=0)
 
             if (eFlag == True):
-                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2])
+                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate(([val],hy),axis=0)
 
@@ -130,12 +130,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False):
             temp = tmp[:,i,j]
             hy = temp.copy()
             if (sFlag == True):
-                val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3])
+                val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate((hy,[val]),axis=0)
 
             if (eFlag == True):
-                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2])
+                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate(([val],hy),axis=0)
 
@@ -148,12 +148,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False):
             temp = vpr[:,i,j]
             hy = temp.copy()
             if (sFlag == True):
-                val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3])
+                val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate((hy,[val]),axis=0)
 
             if (eFlag == True):
-                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2])
+                val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2])
                 #changed from 1 to 0 (-1 should also work), CL
                 hy = np.concatenate(([val],hy),axis=0)
 
diff --git a/tests/test_dload.py b/tests/test_dload.py
@@ -8,8 +8,8 @@
 print('import pyaps3 from {}'.format(pa.__file__))
 print('------------------------------------------------')
 print('test ERA5 data download')
-print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS)-beta.')
-print('      More detailed info can be found on: https://cds-beta.climate.copernicus.eu/how-to-api')
+print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS).')
+print('      More detailed info can be found on: https://cds.climate.copernicus.eu/how-to-api')
 print('      Add your account info to ~/.cdsapirc file.')
 filedir = os.path.join(os.path.dirname(__file__), 'data', 'ERA5')
 pa.ECMWFdload(['20200601','20200901'], hr='14', filedir=filedir, model='ERA5', snwe=(30,40,120,140))

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-cdsapi>=0.7.0`
	`1`	`+cdsapi>=0.7.2`
`2`	`2`	`numpy`
`3`	`3`	`pygrib`
`4`	`4`	`scipy`