Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,14 @@ and start a new "In Progress" section above it.
- Require at least Python 3.11 (drop support for Python 3.8, 3.9 and 3.10 [#370](https://github.com/Open-EO/openeo-python-driver/issues/370))
- Require at least PySTAC 1.10 ([#396](https://github.com/Open-EO/openeo-python-driver/issues/396))
- Replace the unmaintained `openeo_processes` dependency with a minimal, numpy-only in-tree port (`openeo_driver.vendor.processes_python`, adapted from openeo-processes-python 0.0.4) covering only the processes used by the driver. Related to [#152](https://github.com/Open-EO/openeo-python-driver/issues/152).
- Expose raster asset `data_type` and `nodata` metadata in batch job results ([openeo-geotrellis-extensions#863](https://github.com/Open-EO/openeo-geotrellis-extensions/issues/863))
- Replace `DelayedVector` with `DriverVectorCube` in `read_vector`, `get_geometries`, `vector_buffer` and the dummy backend's `aggregate_spatial` (vector files are now loaded through `DriverVectorCube.from_fiona`) and remove the `DelayedVector` handling from `apply_polygon`, `chunk_polygon`, `mask_polygon`, `filter_spatial`, `aggregate_spatial` and `run_udf`
- `read_vector` and `get_geometries` (with `filename`) now return a `DriverVectorCube` (built with the backend's `vector_cube_cls`) instead of a `DelayedVector`.
- Vector files are now loaded eagerly (instead of lazily) and loading errors are reported as `load_url_reading_error` instead of the former GeoJSON parsing errors.
- **Backwards incompatible changes:**
- `run_udf` on the result of `read_vector`/`get_geometries` no longer passes the geometries as feature collection to the UDF, but follows the generic vector cube handling (like `load_uploaded_files`).
- Use `save_result` node id in synchronous download filename hint (content-disposition header), and align filename extension handling with batch jobs ([#548](https://github.com/Open-EO/openeo-python-driver/issues/548))


## 0.141.0

- Extract openeo-geopyspark-driver oriented batch job result metadata handling from generic view layer and allow openeo-aggregator to inject an alternative implementation in context of "STAC 1.1" batch job result metadata style ([Open-EO/openeo-aggregator/204](https://github.com/Open-EO/openeo-aggregator/issues/204))
Expand Down
2 changes: 1 addition & 1 deletion openeo_driver/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.142.0a3"
__version__ = "0.142.0a4"
26 changes: 25 additions & 1 deletion openeo_driver/backend_/_geopyspark.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import os
import urllib.parse
from pathlib import Path
from typing import List, Optional
from typing import List, Optional, Union

import flask
import openeo
Expand Down Expand Up @@ -420,6 +420,15 @@ def _list_job_results_openeo100(
def _asset_object(
job_id, user_id, filename: str, asset_metadata: dict, job_info: BatchJobMetadata, stac11: bool
) -> dict:
def jsonable_nodata(nodata: Optional[float]) -> Union[None, float, str]:
if nodata != nodata:
return "nan"

return nodata

nodata_value = jsonable_nodata(asset_metadata.get("nodata"))
data_type_value = asset_metadata.get("data_type")

result_dict = dict_no_none(
{
"title": asset_metadata.get("title", filename),
Expand All @@ -433,6 +442,8 @@ def _asset_object(
"raster:bands": None if stac11 else asset_metadata.get("raster:bands"),
"file:size": asset_metadata.get("file:size"),
"alternate": asset_metadata.get("alternate"),
"nodata": nodata_value if stac11 else None,
"data_type": data_type_value if stac11 else None,
}
)
if filename.endswith(".model"):
Expand Down Expand Up @@ -469,6 +480,19 @@ def _asset_object(
)
for band in bands
]

raster_bands = result_dict.get("raster:bands")
if raster_bands is None:
if nodata_value is not None or data_type_value is not None:
result_dict["raster:bands"] = [
dict_no_none(data_type=data_type_value, nodata=nodata_value) for _ in bands
]
Comment on lines +487 to +489
else:
for raster_band in raster_bands:
if "nodata" not in raster_band and nodata_value is not None:
raster_band["nodata"] = nodata_value
if "data_type" not in raster_band and data_type_value is not None:
raster_band["data_type"] = data_type_value
else:

def raster_bands(band_index) -> dict:
Expand Down
1 change: 1 addition & 0 deletions openeo_driver/dummy/dummy_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,7 @@ def get_result_assets(self, job_id: str, user_id: str) -> Dict[str, dict]:
"type": "image/tiff; application=geotiff",
"roles": ["data"],
"bands": [Band(name="B02", common_name="blue", wavelength_um=0.665)],
"data_type": "uint16",
"nodata": 123,
"instruments": "MSI",
"bbox": [0.0, 50.0, 5.0, 55.0],
Expand Down
88 changes: 55 additions & 33 deletions tests/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -1912,6 +1912,7 @@ def test_get_job_results_100(self, api100):
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/output.tiff",
"type": "image/tiff; application=geotiff",
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/output.nc",
Expand Down Expand Up @@ -1985,6 +1986,7 @@ def test_get_job_results_100(self, api100):
"proj:code": "EPSG:4326",
"proj:shape": [300, 600],
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.0.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/assets/output.nc",
Expand Down Expand Up @@ -2086,6 +2088,7 @@ def test_get_job_results_110(self, api110):
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/output.tiff",
"type": "image/tiff; application=geotiff",
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/output.nc",
Expand Down Expand Up @@ -2171,6 +2174,7 @@ def test_get_job_results_110(self, api110):
"proj:code": "EPSG:4326",
"proj:shape": [300, 600],
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/assets/output.nc",
Expand Down Expand Up @@ -2320,13 +2324,11 @@ def test_get_job_results_110(self, api110):
)

def test_get_job_results_public_href_asset_100(self, api, backend_implementation):
import numpy as np

results_data = {
"output.tiff": {
BatchJobs.ASSET_PUBLIC_HREF: "http://storage.test/r362/res.tiff?sgn=23432ldf348fl4r349",
"type": "application/tiff",
"nodata": np.nan,
"nodata": float("nan"),
}
}
with self._fresh_job_registry(
Expand Down Expand Up @@ -2358,6 +2360,7 @@ def test_get_job_results_signed_100(self, api100, flask_app, backend_config_over
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/50afb0cad129e61d415278c4ffcd8a83/output.tiff",
"type": "image/tiff; application=geotiff",
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/e28f17365e698783574dd313de0d64cd/output.nc",
Expand Down Expand Up @@ -2505,6 +2508,7 @@ def test_get_job_results_signed_110(self, api110, flask_app, backend_config_over
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/50afb0cad129e61d415278c4ffcd8a83/output.tiff",
"type": "image/tiff; application=geotiff",
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/e28f17365e698783574dd313de0d64cd/output.nc",
Expand Down Expand Up @@ -2654,6 +2658,7 @@ def test_get_job_results_signed_with_expiration_100(self, api100, flask_app, bac
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/fd0ca65e29c6d223da05b2e73a875683/output.tiff?expires=2234",
"type": "image/tiff; application=geotiff",
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
},
"output.nc": {
"href": "http://oeo.net/openeo/1.0.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/3ed7b944c4f9cc88d2c3ef7534a27596/output.nc?expires=2234",
Expand Down Expand Up @@ -2848,6 +2853,7 @@ def test_get_job_results_signed_with_expiration_110(self, api110, flask_app, bac
"proj:code": "EPSG:4326",
"proj:shape": [300, 600],
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
"roles": ["data"],
},
"output.nc": {
Expand Down Expand Up @@ -3012,6 +3018,7 @@ def test_get_job_results_from_stac_1_1_items(self, api110, backend_config_overri
"type": "Polygon",
},
"href": "openEO_20231231T214100Z.tif",
"data_type": "float64",
"nodata": "nan",
"type": "image/tiff; application=geotiff",
"bands": [
Expand Down Expand Up @@ -3053,6 +3060,7 @@ def test_get_job_results_from_stac_1_1_items(self, api110, backend_config_overri
},
"href": "s3://openeo-data-staging-waw4-1/batch_jobs/j-250605095828442799fdde3c29b5b047/openEO_20231231T214100Z_abs.tif",
"output_dir" : "s3://openeo-data-staging-waw4-1/batch_jobs/j-250605095828442799fdde3c29b5b047",
"data_type": "float64",
"nodata": "nan",
"type": "image/tiff; application=geotiff",
"bands": [
Expand Down Expand Up @@ -3112,6 +3120,7 @@ def test_get_job_results_from_stac_1_1_items(self, api110, backend_config_overri
"type": "Polygon",
},
"href": "openEO_20231230T214100Z.tif",
"data_type": "float64",
"nodata": "nan",
"type": "image/tiff; application=geotiff",
"bands": [
Expand Down Expand Up @@ -3158,38 +3167,50 @@ def test_get_job_results_from_stac_1_1_items(self, api110, backend_config_overri
"roles": ["data"],
"title": "openEO_20231230T214100Z.tif",
"type": "image/tiff; application=geotiff",
"nodata": "nan",
"data_type": "float64",
},
'5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z_openEO_rel':{
'href': 'http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/d58330c450ed1695361301efe130faf2/openEO_20231231T214100Z.tif',
'bands': [{
'name': 'LST',
'eo:common_name': 'surface_temperature',
'statistics': {
'maximum': 281.04800415039,
'mean': 259.57087672984,
'minimum': 224.46798706055,
'stddev': 19.598456945276,
'valid_percent': 66.88}
}],
'roles': ['data'],
'title': 'openEO_20231231T214100Z.tif',
'type': 'image/tiff; application=geotiff'
"5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z_openEO_rel": {
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/d58330c450ed1695361301efe130faf2/openEO_20231231T214100Z.tif",
"bands": [
{
"name": "LST",
"eo:common_name": "surface_temperature",
"statistics": {
"maximum": 281.04800415039,
"mean": 259.57087672984,
"minimum": 224.46798706055,
"stddev": 19.598456945276,
"valid_percent": 66.88,
},
}
],
"roles": ["data"],
"title": "openEO_20231231T214100Z.tif",
"type": "image/tiff; application=geotiff",
"nodata": "nan",
"data_type": "float64",
},
'5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z_openEO_abs': {
'href': 'http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/43241f3f756544ac5eb004268c3aa56a/openEO_20231231T214100Z_abs.tif',
'bands': [{
'name': 'LST',
'eo:common_name': 'surface_temperature',
'statistics': {
'maximum': 281.04800415039,
'mean': 259.57087672984,
'minimum': 224.46798706055,
'stddev': 19.598456945276,
'valid_percent': 66.88}
}],
'roles': ['data'],
'title': 'openEO_20231231T214100Z_abs.tif',
'type': 'image/tiff; application=geotiff'
"5d2db643-5cc3-4b27-8ef3-11f7d203b221_2023-12-31T21:41:00Z_openEO_abs": {
"href": "http://oeo.net/openeo/1.1.0/jobs/07024ee9-7847-4b8a-b260-6c879a2b3cdc/results/assets/TXIuVGVzdA==/43241f3f756544ac5eb004268c3aa56a/openEO_20231231T214100Z_abs.tif",
"bands": [
{
"name": "LST",
"eo:common_name": "surface_temperature",
"statistics": {
"maximum": 281.04800415039,
"mean": 259.57087672984,
"minimum": 224.46798706055,
"stddev": 19.598456945276,
"valid_percent": 66.88,
},
}
],
"roles": ["data"],
"title": "openEO_20231231T214100Z_abs.tif",
"type": "image/tiff; application=geotiff",
"nodata": "nan",
"data_type": "float64",
}
}
assert resp.get("item_assets") == {
Expand Down Expand Up @@ -4036,6 +4057,7 @@ def test_get_job_result_item(self, flask_app, api110, backend_config_overrides):
"proj:code": "EPSG:4326",
"proj:shape": [300, 600],
"eo:bands": [{"name": "B02", "common_name": "blue", "center_wavelength": 0.665}],
"raster:bands": [{"data_type": "uint16", "nodata": 123}],
"roles": ["data"],
}
},
Expand Down
Loading