- Created by Jason Wang, last modified on Apr 23, 2023
Hi,
I am tryting downloading daily mean data of ERA-5.
I have downloaded 10m_u_component_of_wind by the below code.
But it runs failed when I try surface_latent_heat_flux or surface_sensible_heat_flux.
downloadERA5.py Expand source
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 6 14:54:38 2021
@author: Dong
"""
import cdsapi
import requests
# CDS API script to use CDS service to retrieve daily ERA5* variables and iterate over
# all months in the specified years.
# Requires:
# 1) the CDS API to be installed and working on your system
# 2) You have agreed to the ERA5 Licence (via the CDS web page)
# 3) Selection of required variable, daily statistic, etc
# Output:
# 1) separate netCDF file for chosen daily statistic/variable for each month
c = cdsapi.Client(timeout=300)
# Uncomment years as required
years = [
'1979'
# ,'1980', '1981',
# '1982', '1983', '1984',
# '1985', '1986', '1987',
# '1988', '1989', '1990',
# '1991', '1992', '1993',
# '1994', '1995', '1996',
# '1997', '1998', '1999',
# '2000', '2001', '2002',
# '2003', '2004', '2005',
# '2006', '2007', '2008',
# '2009', '2010', '2011',
# '2012', '2013', '2014',
# '2015', '2016', '2017',
# '2018', '2019', '2020',
# '2021', '2022'
]
# Retrieve all months for a given year.
months = ['01', '02', '03',
'04', '05', '06',
'07', '08', '09',
'10', '11', '12']
# For valid keywords, see Table 2 of:
# https://datastore.copernicus-climate.eu/documents/app-c3s-daily-era5-statistics/C3S_Application-Documentation_ERA5-daily-statistics-v2.pdf
# select your variable; name must be a valid ERA5 CDS API name.
var = "surface_latent_heat_flux"
# Select the required statistic, valid names given in link above
stat = "daily_mean"
# Loop over years and months
for yr in years:
for mn in months:
result = c.service(
"tool.toolbox.orchestrator.workflow",
params={
"realm": "c3s",
"project": "app-c3s-daily-era5-statistics",
"version": "master",
"kwargs": {
"dataset": "reanalysis-era5-single-levels",
"product_type": "reanalysis",
"variable": var,
"statistic": stat,
"year": yr,
"month": mn,
"time_zone": "UTC+00:0",
"frequency": "1-hourly",
#
# Users can change the output grid resolution and selected area
#
"grid": "0.25/0.25",
"area":{"lat": [-15, 15], "lon": [90, 144]}
},
"workflow_name": "application"
})
# set name of output file for each month (statistic, variable, year, month
file_name = "ERA5_" + stat + "_" + var + "_" + yr + "_" + mn + ".nc"
location=result[0]['location']
res = requests.get(location, stream = True)
print("Writing data to " + file_name)
with open(file_name,'wb') as fh:
for r in res.iter_content(chunk_size = 1024):
fh.write(r)
fh.close()
Then, I run the script in PowerShell 7, the terminal will display
PowerShell 7 Expand source
PS E:\Data\ERA5\surface_latent_heat_flux> python downloadERA5.py
2023-04-23 22:23:18,620 INFO Welcome to the CDS
2023-04-23 22:23:18,620 INFO Sending request to https://cds.climate.copernicus.eu/api/v2/tasks/services/tool/toolbox/orchestrator/workflow/clientid-3bbcfe0badf1489481f3217e8992513e
2023-04-23 22:23:18,976 INFO Request is queued
2023-04-23 22:23:20,224 INFO Request is running
2023-04-23 22:23:53,336 INFO Request is failed
2023-04-23 22:23:53,336 ERROR Message:
2023-04-23 22:23:53,336 ERROR Reason: Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/jsonrequest/requests.py", line 71, in jsonrequestcall
resp = coding.encode(req.callable(*req.args, **req.kwargs), register=encoders, **context)
File "/usr/local/lib/python3.6/dist-packages/cdsworkflows/submit_workflow.py", line 55, in submit_workflow
results = workflow_bare_func(**kwargs)
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 732, in application
data, dataset, variable, pressure_level, product_type, year, month, grid, area_retrieve, time_shift
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 527, in add_necessary_data
data = concat([data, data_to_add], dim='time')
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 557, in concat
non_common_coords = list(reduce(lambda x, y: set(x)^set(y), coords))
NameError: name 'reduce' is not defined
2023-04-23 22:23:53,336 ERROR Traceback (most recent call last):
2023-04-23 22:23:53,336 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 59, in handle_request
2023-04-23 22:23:53,336 ERROR result = cached(context.method, proc, context, context.args, context.kwargs)
2023-04-23 22:23:53,336 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached
2023-04-23 22:23:53,336 ERROR result = proc(context, *context.args, **context.kwargs)
2023-04-23 22:23:53,336 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 124, in __call__
2023-04-23 22:23:53,337 ERROR return p(*args, **kwargs)
2023-04-23 22:23:53,337 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 60, in __call__
2023-04-23 22:23:53,337 ERROR return self.proc(context, *args, **kwargs)
2023-04-23 22:23:53,337 ERROR File "/home/cds/cdsservices/services/workflow.py", line 35, in execute
2023-04-23 22:23:53,337 ERROR raise exceptions.CDSException(True, True, logging + traceback, '', uri)
2023-04-23 22:23:53,337 ERROR cdsclient.exceptions.CDSException: Traceback (most recent call last):
2023-04-23 22:23:53,337 ERROR File "/usr/local/lib/python3.6/dist-packages/jsonrequest/requests.py", line 71, in jsonrequestcall
2023-04-23 22:23:53,337 ERROR resp = coding.encode(req.callable(*req.args, **req.kwargs), register=encoders, **context)
2023-04-23 22:23:53,337 ERROR File "/usr/local/lib/python3.6/dist-packages/cdsworkflows/submit_workflow.py", line 55, in submit_workflow
2023-04-23 22:23:53,337 ERROR results = workflow_bare_func(**kwargs)
2023-04-23 22:23:53,337 ERROR File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 732, in application
2023-04-23 22:23:53,338 ERROR data, dataset, variable, pressure_level, product_type, year, month, grid, area_retrieve, time_shift
2023-04-23 22:23:53,338 ERROR File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 527, in add_necessary_data
2023-04-23 22:23:53,338 ERROR data = concat([data, data_to_add], dim='time')
2023-04-23 22:23:53,338 ERROR File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 557, in concat
2023-04-23 22:23:53,338 ERROR non_common_coords = list(reduce(lambda x, y: set(x)^set(y), coords))
2023-04-23 22:23:53,338 ERROR NameError: name 'reduce' is not defined
Traceback (most recent call last):
File "E:\Data\ERA5\surface_latent_heat_flux\downloadERA5.py", line 66, in <module>
result = c.service(
File "C:\Users\wjs\AppData\Local\Programs\Python\Python310\lib\site-packages\cdsapi\api.py", line 367, in service
result = self._api(
File "C:\Users\wjs\AppData\Local\Programs\Python\Python310\lib\site-packages\cdsapi\api.py", line 507, in _api
raise Exception(
Exception: . Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/jsonrequest/requests.py", line 71, in jsonrequestcall
resp = coding.encode(req.callable(*req.args, **req.kwargs), register=encoders, **context)
File "/usr/local/lib/python3.6/dist-packages/cdsworkflows/submit_workflow.py", line 55, in submit_workflow
results = workflow_bare_func(**kwargs)
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 732, in application
data, dataset, variable, pressure_level, product_type, year, month, grid, area_retrieve, time_shift
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 527, in add_necessary_data
data = concat([data, data_to_add], dim='time')
File "/workflows/c3s/app-c3s-daily-era5-statistics/master/workflows.py", line 557, in concat
non_common_coords = list(reduce(lambda x, y: set(x)^set(y), coords))
NameError: name 'reduce' is not defined
1 Comment
Jason Wang
Thanks to Kevin Marsh! I solve the problem by this website Retrieve daily ERA5/ERA5-Land data using the CDS API
The "realm" line should be
"realm": "user-apps"
as in the example above.
import cdsapi import requests # CDS API script to use CDS service to retrieve daily ERA5* variables and iterate over # all months in the specified years. # Requires: # 1) the CDS API to be installed and working on your system # 2) You have agreed to the ERA5 Licence (via the CDS web page) # 3) Selection of required variable, daily statistic, etc # Output: # 1) separate netCDF file for chosen daily statistic/variable for each month c = cdsapi.Client(timeout=300) # Uncomment years as required years = [ '1979' # ,'1980', '1981', # '1982', '1983', '1984', # '1985', '1986', '1987', # '1988', '1989', '1990', # '1991', '1992', '1993', # '1994', '1995', '1996', # '1997', '1998', '1999', # '2000', '2001', '2002', # '2003', '2004', '2005', # '2006', '2007', '2008', # '2009', '2010', '2011', # '2012', '2013', '2014', # '2015', '2016', '2017', # '2018', '2019', '2020', # '2021' ] # Retrieve all months for a given year. months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'] # For valid keywords, see Table 2 of: # https://datastore.copernicus-climate.eu/documents/app-c3s-daily-era5-statistics/C3S_Application-Documentation_ERA5-daily-statistics-v2.pdf # select your variable; name must be a valid ERA5 CDS API name. var = "surface_net_solar_radiation" # Select the required statistic, valid names given in link above stat = "daily_mean" # Loop over years and months for yr in years: for mn in months: result = c.service( "tool.toolbox.orchestrator.workflow", params={ "realm": "user-apps", "project": "app-c3s-daily-era5-statistics", "version": "master", "kwargs": { "dataset": "reanalysis-era5-single-levels", "product_type": "reanalysis", "variable": var, "statistic": stat, "year": yr, "month": mn, "time_zone": "UTC+00:0", "frequency": "1-hourly", # # Users can change the output grid resolution and selected area # # "grid": "1.0/1.0", # "area":{"lat": [10, 60], "lon": [65, 140]} }, "workflow_name": "application" }) # set name of output file for each month (statistic, variable, year, month file_name = "download_" + stat + "_" + var + "_" + yr + "_" + mn + ".nc" location=result[0]['location'] res = requests.get(location, stream = True) print("Writing data to " + file_name) with open(file_name,'wb') as fh: for r in res.iter_content(chunk_size = 1024): fh.write(r) fh.close()