Hi guys,
I'm submitting a workflow to perform the daily agreggation of temperature_2m on your server side, but after 2 hours, the request failed due to memory issues.
Unfortunattely it will cost to much if I download hourly data and perform this operation locally.
Do you have any advice?
Check my script bellow:
import cdsapi
cw = cdsapi.Client()
with open("workflows/daily_bboxbrazil_Tmax2m.py") as f:
code = f.read()
rw = cw.workflow(code)
print(cw.download(rw))
script of 'daily_bboxbrazil_Tmax2m.py':
import cdstoolbox as ct
@ct.application(title = 'Daily download Tmax2m')
@ct.output.download()
def application():
data = ct.catalogue.retrieve(
'reanalysis-era5-single-levels',
{
'variable': '2m_temperature',
'product_type': 'reanalysis',
'year': ['2010', '2011', '2012', '2013', '2014',
'2015', '2016', '2017', '2018', '2019',
'2020'],
'month': ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'],
'day': ['01','02','03', '04', '05', '06', '07', '08', '09', '10',
'11', '12', '13', '14', '15', '16', '17', '18', '19', '20',
'21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31'],
'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00',
'06:00', '07:00', '08:00', '09:00', '10:00', '11:00',
'12:00', '13:00', '14:00', '15:00', '16:00', '17:00',
'18:00', '19:00', '20:00', '21:00', '22:00', '23:00'],
'area': [5, -74, -35, -34,],
}
)
daily_aggregation = ct.cube.resample(data, how='max', freq='day', keep_attrs = True)
# print(data, daily_aggregation)
return daily_aggregation
4 Comments
José Lucas Safanelli
Ops, here is the console message:
2021-09-16 11:24:06,301 ERROR Traceback (most recent call last): 2021-09-16 11:24:06,301 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 55, in handle_request 2021-09-16 11:24:06,301 ERROR result = cached(context.method, proc, context, context.args, context.kwargs) 2021-09-16 11:24:06,301 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached 2021-09-16 11:24:06,306 ERROR result = proc(context, *context.args, **context.kwargs) 2021-09-16 11:24:06,306 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 118, in __call__ 2021-09-16 11:24:06,307 ERROR return p(*args, **kwargs) 2021-09-16 11:24:06,307 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 59, in __call__ 2021-09-16 11:24:06,308 ERROR return self.proc(context, *args, **kwargs) 2021-09-16 11:24:06,308 ERROR File "/home/cds/cdsservices/services/run_workflow.py", line 36, in execute 2021-09-16 11:24:06,308 ERROR raise exceptions.CDSException(True, True, logging + traceback, '', uri) 2021-09-16 11:24:06,309 ERROR cdsclient.exceptions.CDSException: Traceback (most recent call last): 2021-09-16 11:24:06,309 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 55, in handle_request 2021-09-16 11:24:06,309 ERROR result = cached(context.method, proc, context, context.args, context.kwargs) 2021-09-16 11:24:06,309 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached 2021-09-16 11:24:06,310 ERROR result = proc(context, *context.args, **context.kwargs) 2021-09-16 11:24:06,310 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 118, in __call__ 2021-09-16 11:24:06,310 ERROR return p(*args, **kwargs) 2021-09-16 11:24:06,310 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 59, in __call__ 2021-09-16 11:24:06,311 ERROR return self.proc(context, *args, **kwargs) 2021-09-16 11:24:06,311 ERROR File "/home/cds/cdsservices/services/retrieve.py", line 201, in execute 2021-09-16 11:24:06,311 ERROR **retrieve_kwargs 2021-09-16 11:24:06,311 ERROR File "/opt/cdstoolbox/cdscompute/cdscompute/context.py", line 301, in call 2021-09-16 11:24:06,312 ERROR return c.call(service, *args, **kwargs).value 2021-09-16 11:24:06,312 ERROR File "/opt/cdstoolbox/cdsworkflows/cdsworkflows/future.py", line 76, in value 2021-09-16 11:24:06,312 ERROR raise self._result 2021-09-16 11:24:06,312 ERROR cdsworkflows.error.ClientError: {'traceback': 'Traceback (most recent call last):\n File "/usr/local/lib/python3.6/site-packages/xarray/core/dataset.py", line 1567, in to_netcdf\n invalid_netcdf=invalid_netcdf,\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/api.py", line 1082, in to_netcdf\n dataset, store, writer, encoding=encoding, unlimited_dims=unlimited_dims\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/api.py", line 1128, in dump_to_store\n store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/common.py", line 250, in store\n variables, attributes = self.encode(variables, attributes)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/common.py", line 339, in encode\n variables, attributes = cf_encoder(variables, attributes)\n File "/usr/local/lib/python3.6/site-packages/xarray/conventions.py", line 773, in cf_encoder\n new_vars = {k: encode_cf_variable(v, name=k) for k, v in variables.items()}\n File "/usr/local/lib/python3.6/site-packages/xarray/conventions.py", line 773, in <dictcomp>\n new_vars = {k: encode_cf_variable(v, name=k) for k, v in variables.items()}\n File "/usr/local/lib/python3.6/site-packages/xarray/conventions.py", line 252, in encode_cf_variable\n var = coder.encode(var, name=name)\n File "/usr/local/lib/python3.6/site-packages/xarray/coding/times.py", line 411, in encode\n variable\n File "/usr/local/lib/python3.6/site-packages/xarray/core/common.py", line 1693, in contains_cftime_datetimes\n return _contains_cftime_datetimes(var.data)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 359, in data\n return self.values\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 510, in values\n return _as_array_or_item(self._data)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 272, in _as_array_or_item\n data = np.asarray(data)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 685, in __array__\n self._ensure_cached()\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 682, in _ensure_cached\n self.array = NumpyIndexingAdapter(np.asarray(self.array))\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 655, in __array__\n return np.asarray(self.array, dtype=dtype)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 24, in __getitem__\n key, self.shape, indexing.IndexingSupport.OUTER, self._getitem\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 845, in explicit_indexing_adapter\n result = raw_indexing_method(raw_key.tuple)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 29, in _getitem\n return self.array[key]\n File "/usr/local/lib/python3.6/site-packages/cfgrib/dataset.py", line 336, in __getitem__\n array_field = np.full(array_field_shape, fill_value=np.nan, dtype="float32")\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/numeric.py", line 332, in full\n a = empty(shape, dtype, order)\nMemoryError: Unable to allocate array with shape (96432, 161, 161) and data type float32\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/opt/cds/cdsinf/python/lib/cdsinf/runner/dispatcher.py", line 616, in handle_request\n context.get("method_config", None))\n File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 55, in handle_request\n result = cached(context.method, proc, context, context.args, context.kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached\n result = proc(context, *context.args, **context.kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 118, in __call__\n return p(*args, **kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 59, in __call__\n return self.proc(context, *args, **kwargs)\n File "/home/cds/cdsservices/services/cdm_translate.py", line 72, in execute\n io.dataset_to_netcdf(dataset, out.path)\n File "/opt/cdstoolbox/cdscdm/cdscdm/io.py", line 307, in dataset_to_netcdf\n dataset.to_netcdf(output_path, engine=\'scipy\', **kwargs)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/dataset.py", line 1567, in to_netcdf\n invalid_netcdf=invalid_netcdf,\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/api.py", line 1082, in to_netcdf\n dataset, store, writer, encoding=encoding, unlimited_dims=unlimited_dims\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/api.py", line 1128, in dump_to_store\n store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/common.py", line 250, in store\n variables, attributes = self.encode(variables, attributes)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/common.py", line 339, in encode\n variables, attributes = cf_encoder(variables, attributes)\n File "/usr/local/lib/python3.6/site-packages/xarray/conventions.py", line 771, in cf_encoder\n _update_bounds_encoding(variables)\n File "/usr/local/lib/python3.6/site-packages/xarray/conventions.py", line 413, in _update_bounds_encoding\n ) or contains_cftime_datetimes(v)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/common.py", line 1693, in contains_cftime_datetimes\n return _contains_cftime_datetimes(var.data)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 359, in data\n return self.values\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 510, in values\n return _as_array_or_item(self._data)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 272, in _as_array_or_item\n data = np.asarray(data)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 685, in __array__\n self._ensure_cached()\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 682, in _ensure_cached\n self.array = NumpyIndexingAdapter(np.asarray(self.array))\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 655, in __array__\n return np.asarray(self.array, dtype=dtype)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 24, in __getitem__\n key, self.shape, indexing.IndexingSupport.OUTER, self._getitem\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 845, in explicit_indexing_adapter\n result = raw_indexing_method(raw_key.tuple)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 29, in _getitem\n return self.array[key]\n File "/usr/local/lib/python3.6/site-packages/cfgrib/dataset.py", line 336, in __getitem__\n array_field = np.full(array_field_shape, fill_value=np.nan, dtype="float32")\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/numeric.py", line 332, in full\n a = empty(shape, dtype, order)\nMemoryError: Unable to allocate array with shape (96432, 161, 161) and data type float32\n'}Kevin Marsh
Hi Jose,
I believe the memory error is because you are trying to read the data for all years in one request; for hourly data is is best to loop over months /years, requesting 1 month, subset the data you need and concatenate the results together as you go along. See:
How to calculate daily data from hourly data/process a large amount of data in the Toolbox
However, if you want daily statistics of ERA5 hourly data, there is an application which will do this :
https://cds.climate.copernicus.eu/cdsapp#!/software/app-c3s-daily-era5-statistics?tab=app
and you can call this application via a CDS API script; see:
Retrieve daily ERA5/ERA5-Land data using the CDS API
Hope that helps,
Kevin
José Lucas Safanelli
Dear Kevin, you nailed it! Thanks for the quick support
José Lucas Safanelli
Just out of curiosity, do you know how to change the name of the file when downloading it from the workflow? It uses the CDS 'location' by default.
Some change un this part, perhaps:
rw = cw.workflow(code)
print(cw.download(rw))