Merge pull request #324 from ecmwf/develop

1.0.27

Author: mathleur

docs/Service/Data_Portfolio.md

@@ -1,10 +1,10 @@
 # Data Portfolio
 
-Polytope feature extraction only has access to data that is stored on an FDB. The dataset currently available via Polyope feature extraction is the operational forecast. We plan to add Destination Earth Digital Twin data in the future.
+Polytope feature extraction only has access to data that is stored on an FDB. The datasets currently available via Polyope feature extraction are the operational ECMWF forecast, as well as the data produced by the Destination Earth Extremes and Climate digital twins.
 
 ## Operational Forecast Data
 
-The following values available for each field specified are:
+The following key value pairs are available via Polytope:
 
 * `class` : `od`
 * `stream` : `enfo` `oper`
@@ -18,7 +18,7 @@ If `type` is `enfo`:
 
 * `number` : `0/to/50`
 
-If `levtype` is `pl` or `ml` a `levelist` must be provided:
+If `levtype` is `pl` or `ml`, a `levelist` must be provided:
 
 * `levelist` : `1/to/1000`
 
@@ -42,10 +42,86 @@ If `levtype` is `pl` or `ml` a `levelist` must be provided:
     * `crwe`
     * `ttpha`
 
-For `sfc` most `params` will be available but not all.
+For `sfc`, most `params` will be available but not all.
 
-Only data that is contained in the operational FDB can be requested via Polytope feature extraction, the FDB usually only contains the last two days of forecasts.
+Only data that is contained in the operational FDB can be requested via Polytope feature extraction. The FDB usually only contains the last two days of forecasts.
 
 We sometimes limit the size of requests for area features such as bounding box and polygon to maintain quality of service.
 
 Access to operational data is limited by our release schedule.
+
+
+## Extremes DT Data
+
+The following values available for each field specified are:
+
+* `class` : `d1`
+* `dataset` : `extremes-dt`
+* `stream` : `oper` `wave`
+* `type` : `fc`
+* `levtype` : `sfc` `pl` `hl`
+* `expver` : `0001`
+* `domain` : `g`
+* `step` : `0/to/96`
+
+If `levtype` is `pl`, a `levelist` must be provided:
+
+* `levelist` : `1/to/1000`
+
+If `levtype` is `hl`, a `levelist` must be provided:
+
+* `levtype` : `100`
+
+`pl` and `hl` also only contain a subset of parameters that are available in grid point. These are:
+
+* `pl`
+    * `Geopotential`
+    * `Temperature`
+    * `U component of wind`
+    * `V component of wind`
+    * `Specific humidity`
+    * `Relative humidity`
+* `hl`
+    * `100 metre U wind component`
+    * `100 metre V wind component `
+
+For `sfc` most `params` are available.
+
+For `stream` : `wave` the following parameters are available:
+
+* `Mean zero-crossing wave period`
+* `Significant height of combined wind waves and swell`
+* `Mean wave direction`
+* `Peak wave period`
+* `Mean wave period`
+
+Only Extremes-DT data from the past 15 days can be accessed by users.
+
+
+## Climate DT Data
+
+The following values available for each field specified are:
+
+* `class` : `d1`
+* `dataset` : `climate-dt`
+* `activity` : `ScenarioMIP` `story-nudging` `CMIP6`
+* `model`: `IFS-NEMO`
+* `generation` : `1`
+* `realization`: `1`
+* `resolution`: `standard` `high`
+* `time`: `0000/to/2300`
+* `stream` : `clte` 
+* `type` : `fc`
+* `levtype` : `sfc` `pl` `o2d`
+* `expver` : `0001`
+* `domain` : `g`
+
+If `levtype` is `pl`, a `levelist` must be provided:
+
+* `levelist` : `1/to/1000`
+
+`pl` is currently being scanned and new parameters will become available as time passes. This is also the case for `o2d`.
+
+For `sfc`, most `params` are available.
+
+Currently, only data for `dates` between `2020` and `2050` is available.

docs/Service/Examples/examples.md

@@ -5,4 +5,8 @@
 * <a href="../vertical_profile_example">Vertical Profile</a>
 * <a href="../boundingbox_example">Bounding Box</a>
 * <a href="../trajectory_example">Trajectory</a>
-* <a href="../country_example">Country Cut-Out</a>
+* <a href="../country_example">Country Cut-Out</a>
+
+For examples of Polytope Feature Extraction on Destination Earth Digital Twin Data please visit the following Github Repo: https://github.com/destination-earth-digital-twins/polytope-examples
+
+It contains examples for both the Climate DT and the Extremes DT.

docs/Service/Examples/timeseries_example.ipynb

@@ -73,6 +73,30 @@
     "chart.fig.update_layout(yaxis2={\"title\": \"hPa\"})\n",
     "chart.show(renderer=\"png\")  # Replace with chart.show() in an interactive session!"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Convert to Xarray"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "da = ds.to_xarray()\n",
+    "print(da)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -91,7 +115,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.11.8"
   }
  },
  "nbformat": 4,

docs/Service/Examples/vertical_profile_example.ipynb

@@ -21,11 +21,12 @@ request = {
     "feature" : {
         "type" : "timeseries",
         "points": [[-9.10, 38.78]],
-        "axes": "step",
+        "time_axis": "step",
         "range" : {
             "start" : 0,
             "end" : 360,
         }
+        "axes" : ["latitude", "longitude"]
     },
     "format": "covjson",
 }
@@ -42,18 +43,18 @@ For a timeseries within the `feature` dictionary three fields are required
 
 * `type`
 * `points`
-* `axes`
+* `time_axis`
 
 For a timeseries `type` must be `timeseries`.
 
 `points` must be a nested list with a points containing a latitude and a longitude.
 
-`axes` refers to the axis on which to generate the timeseries. In this case the timeseries is generated across `step` based on the inputted `range`. However if the data requested was a climate dataset the `axess` may be `datetime` denoting that the timeseries is generated across that axis.
+`time_axis` refers to the axis on which to generate the timeseries. In this case the timeseries is generated across `step` based on the inputted `range`. However if the data requested was a climate dataset the `time_axis` may be `datetime` denoting that the timeseries is generated across that axis.
 
 
 ## Optional Fields
 
-`range` is an optional field within `feature`. It refers to the extent of the `axes` on which the timeseries will be generated. In the above case where:
+`range` is an optional field within `feature`. It refers to the extent of the `time_axis` on which the timeseries will be generated. In the above case where:
 
 ```python
     "axes": "step",
@@ -75,7 +76,7 @@ A timeseries across `step` will start at step `0` and end at step `360` with all
 ```
 In this case every second step will be returned if it exists.
 
-As `range` is an optional field it can be left out, however there is not a default value. Instead the user has to include the timeseries `axes` in the main body of the request like below:
+As `range` is an optional field it can be left out, however there is not a default value. Instead the user has to include the timeseries `time_axis` in the main body of the request like below:
 
 ```python
 request = {
@@ -93,7 +94,7 @@ request = {
     "feature" : {
         "type" : "timeseries",
         "points": [[-9.10, 38.78]],
-        "axes": "step",
+        "time_axis": "step",
     },
     "format": "covjson",
 }
@@ -104,3 +105,9 @@ This is equivalent to the first request presented.
 At least one of `range` or `step` must be included in the request, but not both. In this case an error will be provided telling the user that `step` is overspecified.
 
 Conversely at least one of `range` or `step` must be included.
+
+`axes` can also be provided which defines the spatial `axes` on which the request is made. For example if the user provides points in the order `longitude`, `latitude` they can add `axes` : `["longitude", "latitude"]`.
+
+## Note:
+
+Previously the `axes` keyword was used for `time_axis`. We still allow this behavior to satisfy backwards compatibility with previous requests.

docs/Service/Features/timeseries.md

@@ -80,10 +80,21 @@ def check_branching_axes(self, request):
                     (upper, lower, idx) = polytope.extents(ax)
                     if "sfc" in polytope.points[idx]:
                         self.fdb_coordinates.pop("levelist", None)
+
+                if ax == "param":
+                    (upper, lower, idx) = polytope.extents(ax)
+                    if "140251" not in polytope.points[idx]:
+                        self.fdb_coordinates.pop("direction", None)
+                        self.fdb_coordinates.pop("frequency", None)
+                    else:
+                        # special param with direction and frequency
+                        if len(polytope.points[idx]) > 1:
+                            raise ValueError(
+                                "Param 251 is part of a special branching of the datacube. Please request it separately."  # noqa: E501
+                            )
         self.fdb_coordinates.pop("quantile", None)
-        # TODO: When do these not appear??
-        self.fdb_coordinates.pop("direction", None)
-        self.fdb_coordinates.pop("frequency", None)
+        self.fdb_coordinates.pop("year", None)
+        self.fdb_coordinates.pop("month", None)
 
         # NOTE: verify that we also remove the axis object for axes we've removed here
         axes_to_remove = set(self.complete_axes) - set(self.fdb_coordinates.keys())

polytope_feature/datacube/backends/fdb.py

@@ -137,4 +137,5 @@ def unmap_tree_node(self, node, unwanted_path):
     "reduced_ll": "ReducedLatLonMapper",
     "local_regular": "LocalRegularGridMapper",
     "healpix_nested": "NestedHealpixGridMapper",
+    "reduced_gaussian": "ReducedGaussianGridMapper",
 }