DEDT Lumi through ECMWF Polytope API

Hint

You can run this notebook in a live session with Binder.

DEDT Lumi through ECMWF Polytope API#

[1]:

from eodag import EODataAccessGateway

dag = EODataAccessGateway()

dedt_lumi expects additional search parameters to be able to request for data, otherwise search validation will fail.

Check queryables for desired collection:

[2]:

dag.list_queryables(provider="dedt_lumi", collection="DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1")

[2]:
QueryablesDict (17) - additional_properties=False
'end':  typing.Annotated[str,  FieldInfo('default': '1990-01-01T00:00:00Z',  'required': False, ... )] typing.Annotated[
str,
FieldInfo(annotation=NoneType, required=False, default='1990-01-01T00:00:00Z', alias='end_datetime', alias_priority=2, description="Date/time as string in ISO 8601 format (e.g. '2024-06-10T12:00:00Z')")
]
'start':  typing.Annotated[str,  FieldInfo('default': '1990-01-01T00:00:00Z',  'required': False, ... )] typing.Annotated[
str,
FieldInfo(annotation=NoneType, required=False, default='1990-01-01T00:00:00Z', alias='start_datetime', alias_priority=2, description="Date/time as string in ISO 8601 format (e.g. '2024-06-10T12:00:00Z')")
]
'ecmwf_activity':  typing.Annotated[Literal['CMIP6'],  FieldInfo('default': 'CMIP6',  'required': False, ... )] typing.Annotated[
typing.Literal['CMIP6'],
FieldInfo(annotation=NoneType, required=False, default='CMIP6', alias='ecmwf:activity', alias_priority=2, title='activity')
]
'ecmwf_class':  typing.Annotated[Literal['d1'],  FieldInfo('default': 'd1',  'required': False, ... )] typing.Annotated[
typing.Literal['d1'],
FieldInfo(annotation=NoneType, required=False, default='d1', alias='ecmwf:class', alias_priority=2, title='class')
]
'ecmwf_dataset':  typing.Annotated[Literal['climate-dt'],  FieldInfo('default': 'climate-dt',  'required': False, ... )] typing.Annotated[
typing.Literal['climate-dt'],
FieldInfo(annotation=NoneType, required=False, default='climate-dt', alias='ecmwf:dataset', alias_priority=2, title='dataset')
]
'ecmwf_experiment':  typing.Annotated[Literal['hist'],  FieldInfo('default': 'hist',  'required': False, ... )] typing.Annotated[
typing.Literal['hist'],
FieldInfo(annotation=NoneType, required=False, default='hist', alias='ecmwf:experiment', alias_priority=2, title='experiment')
]
'ecmwf_expver':  typing.Annotated[Literal['0001'],  FieldInfo('default': '0001',  'required': False, ... )] typing.Annotated[
typing.Literal['0001'],
FieldInfo(annotation=NoneType, required=False, default='0001', alias='ecmwf:expver', alias_priority=2, title='expver')
]
'ecmwf_generation':  typing.Annotated[Literal['1'],  FieldInfo('default': '1',  'required': False, ... )] typing.Annotated[
typing.Literal['1'],
FieldInfo(annotation=NoneType, required=False, default='1', alias='ecmwf:generation', alias_priority=2, title='generation')
]
'ecmwf_levelist':  typing.Annotated[Literal['1', ...],  FieldInfo( 'required': False, ... )] typing.Annotated[
typing.Literal['1', '10', '100', '1000', '11', '12', '13', '14', '15', '150', '16', '17', '18', '19', '2', '20', '200', '21', '22', '23', '24', '25', '250', '26', '27', '28', '29', '3', '30', '300', '31', '32', '33', '34', '35', '36', '37', '38', '39', '4', '40', '400', '41', '42', '43', '44', '45', '46', '47', '48', '49', '5', '50', '500', '51', '52', '53', '54', '55', '56', '57', '58', '59', '6', '60', '600', '61', '62', '63', '64', '65', '66', '67', '68', '69', '7', '70', '700', '71', '72', '73', '74', '75', '8', '850', '9', '925'],
FieldInfo(annotation=NoneType, required=False, default=None, alias='ecmwf:levelist', alias_priority=2, title='levelist')
]
'ecmwf_levtype':  typing.Annotated[Literal['hl', ...],  FieldInfo( 'required': True, ... )] typing.Annotated[
typing.Literal['hl', 'o2d', 'o3d', 'pl', 'sfc', 'sol'],
FieldInfo(annotation=NoneType, required=True, alias='ecmwf:levtype', alias_priority=2, title='levtype')
]
'ecmwf_model':  typing.Annotated[Literal['IFS-NEMO'],  FieldInfo('default': 'IFS-NEMO',  'required': False, ... )] typing.Annotated[
typing.Literal['IFS-NEMO'],
FieldInfo(annotation=NoneType, required=False, default='IFS-NEMO', alias='ecmwf:model', alias_priority=2, title='model')
]
'ecmwf_param':  typing.Annotated[Literal['129', ...],  FieldInfo( 'required': True, ... )] typing.Annotated[
typing.Literal['129', '130', '131', '132', '133', '134', '135', '137', '141', '144', '146', '147', '148', '151', '157', '159', '164', '165', '166', '167', '168', '169', '172', '175', '176', '177', '178', '179', '180', '181', '182', '186', '187', '188', '212', '228', '228141', '228246', '228247', '235', '246', '260048', '263000', '263001', '263003', '263004', '263008', '263009', '263021', '263022', '263100', '263101', '263121', '263122', '263124', '263500', '263501', '263505', '263506', '263507', '60', '78', '79', '8', '9'],
FieldInfo(annotation=NoneType, required=True, alias='ecmwf:param', alias_priority=2, title='param')
]
'ecmwf_realization':  typing.Annotated[Literal['1'],  FieldInfo('default': '1',  'required': False, ... )] typing.Annotated[
typing.Literal['1'],
FieldInfo(annotation=NoneType, required=False, default='1', alias='ecmwf:realization', alias_priority=2, title='realization')
]
'ecmwf_resolution':  typing.Annotated[Literal['high', ...],  FieldInfo( 'required': True, ... )] typing.Annotated[
typing.Literal['high', 'standard'],
FieldInfo(annotation=NoneType, required=True, alias='ecmwf:resolution', alias_priority=2, title='resolution')
]
'ecmwf_stream':  typing.Annotated[Literal['clte'],  FieldInfo('default': 'clte',  'required': False, ... )] typing.Annotated[
typing.Literal['clte'],
FieldInfo(annotation=NoneType, required=False, default='clte', alias='ecmwf:stream', alias_priority=2, title='stream')
]
'ecmwf_time':  typing.Annotated[Literal['0000', ...],  FieldInfo( 'required': True, ... )] typing.Annotated[
typing.Literal['0000', '0100', '0200', '0300', '0400', '0500', '0600', '0700', '0800', '0900', '1000', '1100', '1200', '1300', '1400', '1500', '1600', '1700', '1800', '1900', '2000', '2100', '2200', '2300'],
FieldInfo(annotation=NoneType, required=True, alias='ecmwf:time', alias_priority=2, title='time')
]
'ecmwf_type':  typing.Annotated[Literal['fc'],  FieldInfo('default': 'fc',  'required': False, ... )] typing.Annotated[
typing.Literal['fc'],
FieldInfo(annotation=NoneType, required=False, default='fc', alias='ecmwf:type', alias_priority=2, title='type')
]

We can see that several parameters are required, and we can choose associated values from returned queryables.

We also include a geometry as EMCWF Polytope API now supports them.

[3]:

result = dag.search(
    provider="dedt_lumi",
    start="2000-01-01",
    collection="DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1",
    geom={'lonmin': 22, 'latmin': 18, 'lonmax': 37, 'latmax': 45},
    **{
        'ecmwf:resolution': 'high',
        'ecmwf:levtype': 'sfc',
        'ecmwf:time': '0000',
        'ecmwf:param': '78',
    }
)
result

[3]:
SearchResult (1)
0  EOProduct(id=DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_ORDERABLE_9d8c402d83e0cbc099b2b00d99612a9608b9eef3, provider=dedt_lumi)
EOProduct
provider: 'dedt_lumi',
collection: 'DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1',
properties["id"]: 'DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_ORDERABLE_9d8c402d83e0cbc099b2b00d99612a9608b9eef3',
properties["start_datetime"]: '2000-01-01T00:00:00.000Z',
properties["end_datetime"]: '2000-01-01T00:00:00.000Z',
properties: (25){
end_datetime: '2000-01-01T00:00:00.000Z',
id: 'DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_ORDERABLE_9d8c402d83e0cbc099b2b00d99612a9608b9eef3',
qs: { 'activity': 'CMIP6' , 'class': 'd1' , 'dataset': 'climate-dt' , 'date': '20000101/to/20000101' , 'experiment': 'hist' , 'expver': '0001' , 'feature': { 'shape': [[18.0 , 22.0 ] , [45.0 , 22.0 ] , [45.0 , 37.0 ] , [18.0 , 37.0 ] , [18.0 , 22.0 ] ] , 'type': 'polygon' } , 'generation': '1' , 'levtype': 'sfc' , 'model': 'IFS-NEMO' , 'param': '78' , 'realization': '1' , 'resolution': 'high' , 'stream': 'clte' , 'time': '0000' , 'type': 'fc' },
start_datetime: '2000-01-01T00:00:00.000Z',
title: 'DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_ORDERABLE_9d8c402d83e0cbc099b2b00d99612a9608b9eef3',
ecmwf:activity: 'CMIP6',
ecmwf:class: 'd1',
ecmwf:dataset: 'climate-dt',
ecmwf:date: '20000101/to/20000101',
ecmwf:experiment: 'hist',
ecmwf:expver: '0001',
ecmwf:feature: { 'shape': [[18.0 , 22.0 ] , [45.0 , 22.0 ] , [45.0 , 37.0 ] , [18.0 , 37.0 ] , [18.0 , 22.0 ] ] , 'type': 'polygon' },
ecmwf:generation: '1',
ecmwf:levtype: 'sfc',
ecmwf:model: 'IFS-NEMO',
ecmwf:param: '78',
ecmwf:realization: '1',
ecmwf:resolution: 'high',
ecmwf:stream: 'clte',
ecmwf:time: '0000',
ecmwf:type: 'fc',
eodag:default_geometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))',
eodag:order_link: 'https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/requests/destination-earth?{"verb": "retrieve", "request": {"activity": "CMIP6", "class": "d1", "dataset": "climate-dt", "date": "20000101/to/20000101", "experiment": "hist", "expver": "0001", "feature": {"shape": [[18.0, 22.0], [45.0, 22.0], [45.0, 37.0], [18.0, 37.0], [18.0, 22.0]], "type": "polygon"}, "generation": "1", "levtype": "sfc", "model": "IFS-NEMO", "param": "78", "realization": "1", "resolution": "high", "stream": "clte", "time": "0000", "type": "fc"} }',
order:status: 'orderable',
product:type: 'climate-dt',
}
assets: (0)
 geometry

Let’s download this product and check its data.

[4]:

path = result[0].download(output_dir='/tmp')

!tree {path}
[Retry #3] Waiting 11.088286s until next order status check (retry every 0.2' for 10')
/tmp/DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_41b57442-9fd2-426d-80fe-46ad0ad07ab7
└── DT_CLIMATE_G1_CMIP6_HIST_IFS_NEMO_R1_41b57442-9fd2-426d-80fe-46ad0ad07ab7.covjson

1 directory, 1 file

When requesting a RoI, ECMWF Polytope will generate a covjson file.

To plot it, we are going to load it into an xarray dataset.

[5]:

import os
import json
from covjsonkit.api import Covjsonkit

import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature

cov_json_path = os.path.join(path, os.listdir(path)[0])

with open(cov_json_path) as f:
    covjson = json.load(f)

# Load CovJson using covjsonkit into an xarray
decoder = Covjsonkit().decode(covjson)
ds = decoder.to_xarray()

ds

[5]:

<xarray.Dataset> Size: 4MB
Dimensions:    (datetimes: 1, number: 1, steps: 1, points: 104351)
Coordinates:
  * datetimes  (datetimes) <U20 80B '2000-01-01 00:00:00Z'
  * number     (number) int64 8B 0
  * steps      (steps) int64 8B 0
  * points     (points) int64 835kB 0 1 2 3 4 ... 104347 104348 104349 104350
    latitude   (points) float64 835kB 18.01 18.01 18.01 ... 44.99 44.99 44.99
    longitude  (points) float64 835kB 22.06 22.15 22.24 ... 36.8 36.89 36.98
    levelist   (points) float64 835kB 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0
Data variables:
    tclw       (datetimes, number, steps, points) float64 835kB 0.0 ... 0.001738
Attributes: (12/15)
    activity:     cmip6
    class:        d1
    dataset:      climate-dt
    experiment:   hist
    expver:       0001
    generation:   1
    ...           ...
    resolution:   high
    stream:       clte
    type:         fc
    number:       0
    step:         0
    date:         2000-01-01 00:00:00Z
[9]:

# Extracting tclw variable from dataset
# Data in the original covjson is organize in points rather than lat/lon variables
# so we plot as scatter.

sp = ds['tclw'].isel(datetimes=0, number=0, steps=0).values
lats = ds['latitude'].values
lons = ds['longitude'].values

fig = plt.figure(figsize=(10,6))
ax = plt.axes(projection=ccrs.PlateCarree())

# Add features
ax.coastlines()
ax.gridlines(draw_labels=True)
ax.add_feature(cfeature.BORDERS, linewidth=0.5)
ax.add_feature(cfeature.LAND, facecolor='lightgray')

sc = ax.scatter(lons, lats, c=sp, s=40, cmap='viridis',
                transform=ccrs.PlateCarree())

plt.colorbar(sc, orientation='vertical', label='kg m^-2')

margin = 2
ax.set_extent([
    lons.min()-margin, lons.max()+margin,
    lats.min()-margin, lats.max()+margin
], crs=ccrs.PlateCarree())

plt.title("Total column cloud liquid water")
plt.show()
../../_images/notebooks_tutos_tuto_dedt_lumi_roi_10_0.png 
[ ]: