From a315b857f9d1b1b77a9d5a57c2ef0ea086c4dadf Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:15:46 +0200 Subject: [PATCH 01/23] chore(gitignore): ignore Python build artifacts --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1e7c9a3..463bd0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ env_mkdocs/* -runs \ No newline at end of file +runs +__pycache__/ +*.pyc +*.pyo +*.pyd +*.whl \ No newline at end of file From e387cd2c313edccff8ec404db93f638b637663e0 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:20:51 +0200 Subject: [PATCH 02/23] feat(metadata): add CWL metadata and generated workflow reference --- cwl-cli/crop.cwl | 31 ++ cwl-cli/norm-diff.cwl | 32 ++ cwl-cli/otsu.cwl | 32 ++ cwl-cli/stac.cwl | 31 ++ cwl-cli/stage-in-released.cwl | 31 ++ cwl-cli/stage-in.cwl | 31 ++ cwl-cli/stage-out-released.cwl | 32 ++ cwl-cli/stage-out.cwl | 32 ++ .../app-water-bodies-cloud-native.cwl | 29 ++ cwl-workflow/app-water-body-cloud-native.cwl | 29 ++ cwl-workflow/app-water-body.cwl | 29 ++ docs/crop.cwl | 31 ++ .../detect_water_body.md | 210 +++++++++++ .../water-bodies.md | 330 ++++++++++++++++++ .../water-bodies.md | 238 +++++++++++++ .../app-water-body/water-bodies.md | 238 +++++++++++++ docs/reference/cwl/index.md | 45 +++ mkdocs.yml | 1 + scripts/generate-cwl-docs.py | 70 ++++ scripts/requirements-cwl-docs.txt | 2 + 20 files changed, 1504 insertions(+) create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md create mode 100644 docs/reference/cwl/index.md create mode 100644 scripts/generate-cwl-docs.py create mode 100644 scripts/requirements-cwl-docs.txt diff --git a/cwl-cli/crop.cwl b/cwl-cli/crop.cwl index 7b4657a..262b00e 100644 --- a/cwl-cli/crop.cwl +++ b/cwl-cli/crop.cwl @@ -1,4 +1,35 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Crop a STAC item band +s:description: Crop a selected raster band to an area of interest. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: crop diff --git a/cwl-cli/norm-diff.cwl b/cwl-cli/norm-diff.cwl index 827f449..304e48d 100644 --- a/cwl-cli/norm-diff.cwl +++ b/cwl-cli/norm-diff.cwl @@ -1,4 +1,36 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Normalized difference +s:description: Compute the normalized difference water index from cropped raster + bands. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: norm_diff diff --git a/cwl-cli/otsu.cwl b/cwl-cli/otsu.cwl index 6339009..ddbdc4c 100644 --- a/cwl-cli/otsu.cwl +++ b/cwl-cli/otsu.cwl @@ -1,4 +1,36 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Otsu threshold +s:description: Detect water bodies by applying the Otsu threshold to a water + index raster. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: otsu diff --git a/cwl-cli/stac.cwl b/cwl-cli/stac.cwl index f8413f4..bd4712d 100644 --- a/cwl-cli/stac.cwl +++ b/cwl-cli/stac.cwl @@ -1,4 +1,35 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: STAC catalog +s:description: Create a STAC catalog for the water body detection results. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: stac diff --git a/cwl-cli/stage-in-released.cwl b/cwl-cli/stage-in-released.cwl index d2b705b..ee34f37 100644 --- a/cwl-cli/stage-in-released.cwl +++ b/cwl-cli/stage-in-released.cwl @@ -1,4 +1,35 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Stage-in STAC assets (released container) +s:description: Download the assets of a STAC item for local processing. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: main diff --git a/cwl-cli/stage-in.cwl b/cwl-cli/stage-in.cwl index bdff33d..38e4482 100644 --- a/cwl-cli/stage-in.cwl +++ b/cwl-cli/stage-in.cwl @@ -1,4 +1,35 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Stage-in STAC assets +s:description: Download the assets of a STAC item for local processing. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: main diff --git a/cwl-cli/stage-out-released.cwl b/cwl-cli/stage-out-released.cwl index 105bb8e..ed77552 100644 --- a/cwl-cli/stage-out-released.cwl +++ b/cwl-cli/stage-out-released.cwl @@ -1,4 +1,36 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Stage-out STAC results (released container) +s:description: Upload the results catalog and its assets to S3-compatible + storage. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool diff --git a/cwl-cli/stage-out.cwl b/cwl-cli/stage-out.cwl index 128347b..9a31ea9 100644 --- a/cwl-cli/stage-out.cwl +++ b/cwl-cli/stage-out.cwl @@ -1,4 +1,36 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Stage-out STAC results +s:description: Upload the results catalog and its assets to S3-compatible + storage. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index dbfc246..0b9e719 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -2,6 +2,35 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ s:softwareVersion: 1.4.1 +# Training-example authors and creation date mirror codemeta.json. +s:name: Water bodies detection based on NDWI and otsu threshold +s:description: Water bodies detection based on NDWI and otsu threshold applied + to Sentinel-2 COG STAC items +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index f466381..04d312e 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -2,6 +2,35 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ s:softwareVersion: 1.0.0 +# Training-example authors and creation date mirror codemeta.json. +s:name: Water bodies detection based on NDWI and the otsu threshold +s:description: Water bodies detection based on NDWI and otsu threshold applied + to a single Sentinel-2 COG STAC item +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index 39da7b5..96f0fce 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -2,6 +2,35 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ s:softwareVersion: 1.0.0 +# Training-example authors and creation date mirror codemeta.json. +s:name: Water body detection based on NDWI and the otsu threshold +s:description: Water bodies detection based on NDWI and otsu threshold applied + to Sentinel-2 or Landsat-9 staged acquisitions +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: diff --git a/docs/crop.cwl b/docs/crop.cwl index 461034b..9e2d753 100644 --- a/docs/crop.cwl +++ b/docs/crop.cwl @@ -1,4 +1,35 @@ cwlVersion: v1.2 +$namespaces: + s: https://schema.org/ +# Training-example authors and creation date mirror codemeta.json. +s:name: Crop a STAC item band +s:description: Crop a selected raster band to an area of interest. +s:dateCreated: '2022-09-01' +s:license: + s:name: Creative Commons Attribution-ShareAlike 4.0 International + s:url: https://creativecommons.org/licenses/by-sa/4.0/ + s:identifier: CC-BY-SA-4.0 +s:softwareHelp: + s:name: Mastering Earth Observation Application Packaging with CWL + s:url: https://eoap.github.io/mastering-app-package +s:publisher: + s:name: EO Application Packaging + s:identifier: https://github.com/eoap +s:author: +- s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME +- s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME +s:softwareRequirements: +- container runtime +- cwl runner +s:softwareVersion: 1.1.0 class: CommandLineTool id: crop diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md new file mode 100644 index 0000000..1b8e7ba --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md @@ -0,0 +1,210 @@ +# Water bodies detection based on NDWI and otsu threshold v1.4.1 + +Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items + +> This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) +> +> 2022-09-01 - 2026-10-10T18:20:08.682 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) + +## Project Team + +### Authors + +| Name | Email | Organization | Role | Identifier | +|------|-------|--------------|------|------------| +| Doe, Jane | [jane.doe@acme.earth](mailto:jane.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | +| Doe, John | [john.doe@acme.earth](mailto:john.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | + + +### Contributors + +The are no contributors for this project. + + +## Mastering Earth Observation Application Packaging with CWL + +Mastering Earth Observation Application Packaging with CWL can be found on [https://eoap.github.io/mastering-app-package](https://eoap.github.io/mastering-app-package). + + +## Runtime environment + +### Supported Operating Systems + +The are no Supported Operating Systems specified for this project. + + +### Requirements + +- [container runtime](container runtime) +- [cwl runner](cwl runner) + + + +--- + + +## detect_water_body + +### CWL Class + +[Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) + +### Requirements + +* [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) + +### Inputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | bands used for the NDWI | +| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | STAC item | + + +### Steps + +| Id | Runs | Label | Doc | +|----|------|-------|-----| +| [node_crop](#crop) | `#crop` | None | None | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | +| [node_otsu](#otsu) | `#otsu` | None | None | + + +### Outputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | None | None | + + +### OGC API - Processes + +When `detect_water_body` [Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) is exposed through [OGC API - Processes - Part 1: Core](https://docs.ogc.org/is/18-062r2/18-062r2.html), `inputs` and `outputs` fields below represent the interface of the [getProcessDescription](https://developer.ogc.org/api/processes/index.html#tag/ProcessDescription/operation/getProcessDescription) API. + + + +#### Inputs + +![detect_water_body OGC API Processes JSON Inputs schema](./detect_water_body/ogc_processes_inputs.svg "detect_water_body diagram") + +#### Outputs + +![detect_water_body OGC API Processes JSON Outputs schema](./detect_water_body/ogc_processes_outputs.svg "detect_water_body diagram") + + +### UML Diagrams + + +#### Activity diagram + +Learn more about the [Activity diagram](https://en.wikipedia.org/wiki/Activity_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/activity.svg "detect_water_body Activity diagram") + +#### Component diagram + +Learn more about the [Component diagram](https://en.wikipedia.org/wiki/Component_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/component.svg "detect_water_body Component diagram") + +#### Class diagram + +Learn more about the [Class diagram](https://en.wikipedia.org/wiki/Class_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/class.svg "detect_water_body Class diagram") + +#### Sequence diagram + +Learn more about the [Sequence diagram](https://en.wikipedia.org/wiki/Sequence_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/sequence.svg "detect_water_body Sequence diagram") + +#### State diagram + +Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/state.svg "detect_water_body State diagram") + + +### Run in step + +`node_crop` + + + +## crop + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--aoi \ +--epsg \ +--band +``` + +### Run in step + +`node_normalized_difference` + + + +## norm_diff + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--rasters +``` + +### Run in step + +`node_otsu` + + + +## otsu + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `raster` | `--raster` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--raster +``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md new file mode 100644 index 0000000..0df6393 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md @@ -0,0 +1,330 @@ +# Water bodies detection based on NDWI and otsu threshold v1.4.1 + +Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items + +> This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) +> +> 2022-09-01 - 2026-10-10T18:20:08.520 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) + +## Project Team + +### Authors + +| Name | Email | Organization | Role | Identifier | +|------|-------|--------------|------|------------| +| Doe, Jane | [jane.doe@acme.earth](mailto:jane.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | +| Doe, John | [john.doe@acme.earth](mailto:john.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | + + +### Contributors + +The are no contributors for this project. + + +## Mastering Earth Observation Application Packaging with CWL + +Mastering Earth Observation Application Packaging with CWL can be found on [https://eoap.github.io/mastering-app-package](https://eoap.github.io/mastering-app-package). + + +## Runtime environment + +### Supported Operating Systems + +The are no Supported Operating Systems specified for this project. + + +### Requirements + +- [container runtime](container runtime) +- [cwl runner](cwl runner) + + + +--- + + +## water-bodies + +### CWL Class + +[Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) + +### Requirements + +* [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) +* [SubworkflowFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#SubworkflowFeatureRequirement) + +### Inputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | +| `stac_items` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Sentinel-2 STAC items | list of Sentinel-2 COG STAC items | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | + + +### Steps + +| Id | Runs | Label | Doc | +|----|------|-------|-----| +| [node_water_bodies](#detect_water_body) | `#detect_water_body` | None | None | +| [node_stac](#stac) | `#stac` | None | None | + + +### Outputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | + + +### OGC API - Processes + +When `water-bodies` [Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) is exposed through [OGC API - Processes - Part 1: Core](https://docs.ogc.org/is/18-062r2/18-062r2.html), `inputs` and `outputs` fields below represent the interface of the [getProcessDescription](https://developer.ogc.org/api/processes/index.html#tag/ProcessDescription/operation/getProcessDescription) API. + + + +#### Inputs + +![water-bodies OGC API Processes JSON Inputs schema](./water-bodies/ogc_processes_inputs.svg "water-bodies diagram") + +#### Outputs + +![water-bodies OGC API Processes JSON Outputs schema](./water-bodies/ogc_processes_outputs.svg "water-bodies diagram") + + +### UML Diagrams + + +#### Activity diagram + +Learn more about the [Activity diagram](https://en.wikipedia.org/wiki/Activity_diagram) below. + +![water-bodies flow diagram](./water-bodies/activity.svg "water-bodies Activity diagram") + +#### Component diagram + +Learn more about the [Component diagram](https://en.wikipedia.org/wiki/Component_diagram) below. + +![water-bodies flow diagram](./water-bodies/component.svg "water-bodies Component diagram") + +#### Class diagram + +Learn more about the [Class diagram](https://en.wikipedia.org/wiki/Class_diagram) below. + +![water-bodies flow diagram](./water-bodies/class.svg "water-bodies Class diagram") + +#### Sequence diagram + +Learn more about the [Sequence diagram](https://en.wikipedia.org/wiki/Sequence_diagram) below. + +![water-bodies flow diagram](./water-bodies/sequence.svg "water-bodies Sequence diagram") + +#### State diagram + +Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram) below. + +![water-bodies flow diagram](./water-bodies/state.svg "water-bodies State diagram") + + +### Run in step + +`node_water_bodies` + + + +## detect_water_body + +### CWL Class + +[Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) + +### Requirements + +* [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) + +### Inputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | bands used for the NDWI | +| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | STAC item | + + +### Steps + +| Id | Runs | Label | Doc | +|----|------|-------|-----| +| [node_crop](#crop) | `#crop` | None | None | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | +| [node_otsu](#otsu) | `#otsu` | None | None | + + +### Outputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | None | None | + + +### OGC API - Processes + +When `detect_water_body` [Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) is exposed through [OGC API - Processes - Part 1: Core](https://docs.ogc.org/is/18-062r2/18-062r2.html), `inputs` and `outputs` fields below represent the interface of the [getProcessDescription](https://developer.ogc.org/api/processes/index.html#tag/ProcessDescription/operation/getProcessDescription) API. + + + +#### Inputs + +![detect_water_body OGC API Processes JSON Inputs schema](./detect_water_body/ogc_processes_inputs.svg "detect_water_body diagram") + +#### Outputs + +![detect_water_body OGC API Processes JSON Outputs schema](./detect_water_body/ogc_processes_outputs.svg "detect_water_body diagram") + + +### UML Diagrams + + +#### Activity diagram + +Learn more about the [Activity diagram](https://en.wikipedia.org/wiki/Activity_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/activity.svg "detect_water_body Activity diagram") + +#### Component diagram + +Learn more about the [Component diagram](https://en.wikipedia.org/wiki/Component_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/component.svg "detect_water_body Component diagram") + +#### Class diagram + +Learn more about the [Class diagram](https://en.wikipedia.org/wiki/Class_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/class.svg "detect_water_body Class diagram") + +#### Sequence diagram + +Learn more about the [Sequence diagram](https://en.wikipedia.org/wiki/Sequence_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/sequence.svg "detect_water_body Sequence diagram") + +#### State diagram + +Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram) below. + +![detect_water_body flow diagram](./detect_water_body/state.svg "detect_water_body State diagram") + + +### Run in step + +`node_crop` + + + +## crop + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--aoi \ +--epsg \ +--band +``` + +### Run in step + +`node_normalized_difference` + + + +## norm_diff + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--rasters +``` + +### Run in step + +`node_otsu` + + + +## otsu + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `raster` | `--raster` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--raster +``` + + + +### Run in step + +`node_stac` + + + +## stac + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--item` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--item \ +--rasters +``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md new file mode 100644 index 0000000..78303d2 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md @@ -0,0 +1,238 @@ +# Water bodies detection based on NDWI and the otsu threshold v1.0.0 + +Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item + +> This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) +> +> 2022-09-01 - 2026-10-10T18:20:11.475 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) + +## Project Team + +### Authors + +| Name | Email | Organization | Role | Identifier | +|------|-------|--------------|------|------------| +| Doe, Jane | [jane.doe@acme.earth](mailto:jane.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | +| Doe, John | [john.doe@acme.earth](mailto:john.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | + + +### Contributors + +The are no contributors for this project. + + +## Mastering Earth Observation Application Packaging with CWL + +Mastering Earth Observation Application Packaging with CWL can be found on [https://eoap.github.io/mastering-app-package](https://eoap.github.io/mastering-app-package). + + +## Runtime environment + +### Supported Operating Systems + +The are no Supported Operating Systems specified for this project. + + +### Requirements + +- [container runtime](container runtime) +- [cwl runner](cwl runner) + + + +--- + + +## water-bodies + +### CWL Class + +[Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) + +### Requirements + +* [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) + +### Inputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | +| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item reference | Reference to a STAC item | + + +### Steps + +| Id | Runs | Label | Doc | +|----|------|-------|-----| +| [node_crop](#crop) | `#crop` | None | None | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | +| [node_otsu](#otsu) | `#otsu` | None | None | +| [node_stac](#stac) | `#stac` | None | None | + + +### Outputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | + + +### OGC API - Processes + +When `water-bodies` [Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) is exposed through [OGC API - Processes - Part 1: Core](https://docs.ogc.org/is/18-062r2/18-062r2.html), `inputs` and `outputs` fields below represent the interface of the [getProcessDescription](https://developer.ogc.org/api/processes/index.html#tag/ProcessDescription/operation/getProcessDescription) API. + + + +#### Inputs + +![water-bodies OGC API Processes JSON Inputs schema](./water-bodies/ogc_processes_inputs.svg "water-bodies diagram") + +#### Outputs + +![water-bodies OGC API Processes JSON Outputs schema](./water-bodies/ogc_processes_outputs.svg "water-bodies diagram") + + +### UML Diagrams + + +#### Activity diagram + +Learn more about the [Activity diagram](https://en.wikipedia.org/wiki/Activity_diagram) below. + +![water-bodies flow diagram](./water-bodies/activity.svg "water-bodies Activity diagram") + +#### Component diagram + +Learn more about the [Component diagram](https://en.wikipedia.org/wiki/Component_diagram) below. + +![water-bodies flow diagram](./water-bodies/component.svg "water-bodies Component diagram") + +#### Class diagram + +Learn more about the [Class diagram](https://en.wikipedia.org/wiki/Class_diagram) below. + +![water-bodies flow diagram](./water-bodies/class.svg "water-bodies Class diagram") + +#### Sequence diagram + +Learn more about the [Sequence diagram](https://en.wikipedia.org/wiki/Sequence_diagram) below. + +![water-bodies flow diagram](./water-bodies/sequence.svg "water-bodies Sequence diagram") + +#### State diagram + +Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram) below. + +![water-bodies flow diagram](./water-bodies/state.svg "water-bodies State diagram") + + +### Run in step + +`node_crop` + + + +## crop + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--aoi \ +--epsg \ +--band +``` + +### Run in step + +`node_normalized_difference` + + + +## norm_diff + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--rasters +``` + +### Run in step + +`node_otsu` + + + +## otsu + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `raster` | `--raster` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--raster +``` + +### Run in step + +`node_stac` + + + +## stac + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `rasters` | `--water-body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--water-body +``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md new file mode 100644 index 0000000..d45b113 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md @@ -0,0 +1,238 @@ +# Water body detection based on NDWI and the otsu threshold v1.0.0 + +Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or Landsat-9 staged acquisitions + +> This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) +> +> 2022-09-01 - 2026-10-10T18:20:13.780 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) + +## Project Team + +### Authors + +| Name | Email | Organization | Role | Identifier | +|------|-------|--------------|------|------------| +| Doe, Jane | [jane.doe@acme.earth](mailto:jane.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | +| Doe, John | [john.doe@acme.earth](mailto:john.doe@acme.earth) | [ACME](None) | [N/A](None) | [None](None) | + + +### Contributors + +The are no contributors for this project. + + +## Mastering Earth Observation Application Packaging with CWL + +Mastering Earth Observation Application Packaging with CWL can be found on [https://eoap.github.io/mastering-app-package](https://eoap.github.io/mastering-app-package). + + +## Runtime environment + +### Supported Operating Systems + +The are no Supported Operating Systems specified for this project. + + +### Requirements + +- [container runtime](container runtime) +- [cwl runner](cwl runner) + + + +--- + + +## water-bodies + +### CWL Class + +[Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) + +### Requirements + +* [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) + +### Inputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | +| `item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | STAC item reference | Reference to a STAC item | + + +### Steps + +| Id | Runs | Label | Doc | +|----|------|-------|-----| +| [node_crop](#crop) | `#crop` | None | None | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | +| [node_otsu](#otsu) | `#otsu` | None | None | +| [node_stac](#stac) | `#stac` | None | None | + + +### Outputs + +| Id | Type | Label | Doc | +|----|------|-------|-----| +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | + + +### OGC API - Processes + +When `water-bodies` [Workflow](https://www.commonwl.org/v1.2/Workflow.html#Workflow) is exposed through [OGC API - Processes - Part 1: Core](https://docs.ogc.org/is/18-062r2/18-062r2.html), `inputs` and `outputs` fields below represent the interface of the [getProcessDescription](https://developer.ogc.org/api/processes/index.html#tag/ProcessDescription/operation/getProcessDescription) API. + + + +#### Inputs + +![water-bodies OGC API Processes JSON Inputs schema](./water-bodies/ogc_processes_inputs.svg "water-bodies diagram") + +#### Outputs + +![water-bodies OGC API Processes JSON Outputs schema](./water-bodies/ogc_processes_outputs.svg "water-bodies diagram") + + +### UML Diagrams + + +#### Activity diagram + +Learn more about the [Activity diagram](https://en.wikipedia.org/wiki/Activity_diagram) below. + +![water-bodies flow diagram](./water-bodies/activity.svg "water-bodies Activity diagram") + +#### Component diagram + +Learn more about the [Component diagram](https://en.wikipedia.org/wiki/Component_diagram) below. + +![water-bodies flow diagram](./water-bodies/component.svg "water-bodies Component diagram") + +#### Class diagram + +Learn more about the [Class diagram](https://en.wikipedia.org/wiki/Class_diagram) below. + +![water-bodies flow diagram](./water-bodies/class.svg "water-bodies Class diagram") + +#### Sequence diagram + +Learn more about the [Sequence diagram](https://en.wikipedia.org/wiki/Sequence_diagram) below. + +![water-bodies flow diagram](./water-bodies/sequence.svg "water-bodies Sequence diagram") + +#### State diagram + +Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram) below. + +![water-bodies flow diagram](./water-bodies/state.svg "water-bodies State diagram") + + +### Run in step + +`node_crop` + + + +## crop + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | +| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--aoi \ +--epsg \ +--band +``` + +### Run in step + +`node_normalized_difference` + + + +## norm_diff + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--rasters +``` + +### Run in step + +`node_otsu` + + + +## otsu + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `raster` | `--raster` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--raster +``` + +### Run in step + +`node_stac` + + + +## stac + +### CWL Class + +[CommandLineTool](https://www.commonwl.org/v1.2/CommandLineTool.html#CommandLineTool) + +### Inputs + +| Id | Option | Type | +|----|------|-------| +| `item` | `--input-item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | +| `rasters` | `--water-body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | + +### Execution usage example: + +``` +python -m app \ +--input-item \ +--water-body +``` diff --git a/docs/reference/cwl/index.md b/docs/reference/cwl/index.md new file mode 100644 index 0000000..28b2f02 --- /dev/null +++ b/docs/reference/cwl/index.md @@ -0,0 +1,45 @@ +# Generated CWL reference + +These pages are generated with +[Transpiler-Mate cwl2markdown](https://github.com/transpiler-mate/cwl2markdown). +Output directories follow the source path so workflows with the same ID do not +overwrite one another. + +| CWL source | Workflow reference | +| --- | --- | +| `cwl-workflow/app-water-body.cwl` | [Staged acquisition](cwl-workflow/app-water-body/water-bodies.md) | +| `cwl-workflow/app-water-body-cloud-native.cwl` | [Single cloud-native acquisition](cwl-workflow/app-water-body-cloud-native/water-bodies.md) | +| `cwl-workflow/app-water-bodies-cloud-native.cwl` | [Multiple cloud-native acquisitions](cwl-workflow/app-water-bodies-cloud-native/water-bodies.md), [nested detection workflow](cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md) | + +## Regenerate + +Run from the repository root using Python 3.12 or later: + +```sh +python -m venv /tmp/mastering-cwl-docs +/tmp/mastering-cwl-docs/bin/python -m pip install -r scripts/requirements-cwl-docs.txt +/tmp/mastering-cwl-docs/bin/python scripts/generate-cwl-docs.py +``` + +The script invokes the plugin on all 12 tracked `.cwl` files. The runtime parses +each document and validates its Schema.org metadata. The nine standalone +`CommandLineTool` documents have no workflows, which the plugin reports as an +unsupported input; the script reports these as skips. Command-line tools within +the workflow graphs appear in the generated workflow pages. + +The script supplies a compatibility fallback for cwl2markdown 0.2.1, which asks +for `index.md` but packages `index.md.jinja`, and renames rendered `.md.jinja` +output to `.md`. + +## Metadata and output limitations + +The creation date and illustrative Jane/John Doe author records come from the +training repository's `codemeta.json`. They are example metadata, not verified +authorship. The license follows `LICENSE.md` and the README (CC-BY-SA-4.0); +CodeMeta currently declares a different license. Existing workflow versions are +preserved; standalone tools use CodeMeta's application version. + +The generated pages retain upstream template output, including generation +timestamps, links for optional fields rendered as `None`, and references to UML +and OGC API schema images. cwl2markdown does not generate those images; they +require separate transpilers and renderers. diff --git a/mkdocs.yml b/mkdocs.yml index 76b7cc7..44b01ac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,6 +111,7 @@ nav: - Application benchmarking: benchmarking/calrissian-benchmark.md - FAIR Application Packages: fair/best-practice.md - Reference: + - Generated CWL reference: reference/cwl/index.md - OGC Application Package Best Practice: reference/ogc-ap-bp.md - CWL CommandLineTool: reference/cwl-commandlinetool.md - CWL Workflow: reference/cwl-workflow.md diff --git a/scripts/generate-cwl-docs.py b/scripts/generate-cwl-docs.py new file mode 100644 index 0000000..1f09feb --- /dev/null +++ b/scripts/generate-cwl-docs.py @@ -0,0 +1,70 @@ +"""Run cwl2markdown on every tracked CWL document without output collisions.""" + +from pathlib import Path +import subprocess +import sys + + +# cwl2markdown 0.2.1 requests index.md but ships index.md.jinja. Resolve the +# packaged template without modifying the installed third-party package. +RENDER = """ +from jinja2 import PackageLoader, TemplateNotFound +from transpiler_mate.runtime.cli import main + +original = PackageLoader.get_source +def get_source(self, environment, template): + try: + return original(self, environment, template) + except TemplateNotFound: + if self.package_name == 'cwl2markdown' and template == 'index.md': + return original(self, environment, 'index.md.jinja') + raise +PackageLoader.get_source = get_source +main() +""" + + +def main(): + root = Path(__file__).resolve().parents[1] + output_root = root / "docs/reference/cwl" + sources = subprocess.check_output( + ["git", "ls-files", "-z", "--", "*.cwl"], cwd=root + ).decode().split("\0") + failures = [] + page_count = 0 + tool_count = 0 + for source in sorted(filter(None, sources)): + output = output_root / Path(source).with_suffix("") + result = subprocess.run( + [sys.executable, "-c", RENDER, "cwl2markdown", "--output", str(output), source], + cwd=root, + capture_output=True, + text=True, + ) + if result.returncode: + if "Workflow(s) found in input" in result.stderr: + tool_count += 1 + print(f"SKIP: {source}: standalone tool (metadata validated)") + continue + failures.append(source) + print(f"FAIL: {source}\n{result.stdout}{result.stderr}", file=sys.stderr) + continue + pages = sorted(output.glob("*.md.jinja")) + for page in pages: + # These files are already rendered Markdown, despite their suffix. + page.write_text( + "\n".join(line.rstrip() for line in page.read_text().splitlines()).rstrip() + + "\n" + ) + page.rename(page.with_suffix("")) + page_count += len(pages) + print(f"OK: {source}: {len(pages)} workflow page(s)") + print( + f"Generated {page_count} workflow pages; {tool_count} standalone tools; " + f"{len(failures)} failures." + ) + return bool(failures) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/requirements-cwl-docs.txt b/scripts/requirements-cwl-docs.txt new file mode 100644 index 0000000..d2775a7 --- /dev/null +++ b/scripts/requirements-cwl-docs.txt @@ -0,0 +1,2 @@ +cwl2markdown==0.2.1 +transpiler-mate-runtime==1.3.0 From 23851c7aeb5b105bd06b97cd8dd7afca2e4060b5 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:35:56 +0200 Subject: [PATCH 03/23] chore(gitignore): ignore generated PlantUML sources --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 463bd0b..2b01dc2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ __pycache__/ *.pyc *.pyo *.pyd -*.whl \ No newline at end of file +*.whl +*.puml \ No newline at end of file From b5c01561d625b7a71679d3ba37bbf5032ea5ec53 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:35:56 +0200 Subject: [PATCH 04/23] fix(cwl): resolve EOAP validation findings and refresh reference diagrams --- cwl-cli/crop.cwl | 63 +++++++++------- cwl-cli/norm-diff.cwl | 34 +++++---- cwl-cli/otsu.cwl | 36 ++++----- cwl-cli/stac.cwl | 35 +++++---- cwl-cli/stage-in-released.cwl | 36 +++++---- cwl-cli/stage-in.cwl | 36 +++++---- cwl-cli/stage-out-released.cwl | 45 ++++++++---- cwl-cli/stage-out.cwl | 45 ++++++++---- .../app-water-bodies-cloud-native.cwl | 73 ++++++++++++++----- cwl-workflow/app-water-body-cloud-native.cwl | 65 ++++++++++++----- cwl-workflow/app-water-body.cwl | 65 ++++++++++++----- docs/crop.cwl | 63 +++++++++------- .../detect_water_body.md | 18 ++--- .../detect_water_body/activity.svg | 1 + .../detect_water_body/class.svg | 1 + .../detect_water_body/component.svg | 1 + .../ogc_processes_inputs.svg | 1 + .../ogc_processes_outputs.svg | 1 + .../detect_water_body/sequence.svg | 1 + .../detect_water_body/state.svg | 1 + .../water-bodies.md | 24 +++--- .../water-bodies/activity.svg | 1 + .../water-bodies/class.svg | 1 + .../water-bodies/component.svg | 1 + .../water-bodies/ogc_processes_inputs.svg | 1 + .../water-bodies/ogc_processes_outputs.svg | 1 + .../water-bodies/sequence.svg | 1 + .../water-bodies/state.svg | 1 + .../water-bodies.md | 12 +-- .../water-bodies/activity.svg | 1 + .../water-bodies/class.svg | 1 + .../water-bodies/component.svg | 1 + .../water-bodies/ogc_processes_inputs.svg | 1 + .../water-bodies/ogc_processes_outputs.svg | 1 + .../water-bodies/sequence.svg | 1 + .../water-bodies/state.svg | 1 + .../app-water-body/water-bodies.md | 12 +-- .../app-water-body/water-bodies/activity.svg | 1 + .../app-water-body/water-bodies/class.svg | 1 + .../app-water-body/water-bodies/component.svg | 1 + .../water-bodies/ogc_processes_inputs.svg | 1 + .../water-bodies/ogc_processes_outputs.svg | 1 + .../app-water-body/water-bodies/sequence.svg | 1 + .../app-water-body/water-bodies/state.svg | 1 + docs/reference/cwl/index.md | 36 ++++++++- validation/staging-cloud-native.json | 7 ++ validation/staging-detection.json | 4 + validation/staging-staged.json | 11 +++ 48 files changed, 495 insertions(+), 253 deletions(-) create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/activity.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/component.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_outputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/sequence.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/state.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/activity.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/component.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_outputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/sequence.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/state.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/activity.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/component.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_outputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/sequence.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/state.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/activity.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/component.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_outputs.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/sequence.svg create mode 100644 docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/state.svg create mode 100644 validation/staging-cloud-native.json create mode 100644 validation/staging-detection.json create mode 100644 validation/staging-staged.json diff --git a/cwl-cli/crop.cwl b/cwl-cli/crop.cwl index 262b00e..8cead2e 100644 --- a/cwl-cli/crop.cwl +++ b/cwl-cli/crop.cwl @@ -16,34 +16,33 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool id: crop requirements: - InlineJavascriptRequirement: {} - EnvVarRequirement: - envDef: - PYTHONPATH: /app - ResourceRequirement: - coresMax: 1 - ramMax: 512 - NetworkAccess: - networkAccess: true -hints: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PYTHONPATH: /app + ResourceRequirement: + coresMax: 1 + ramMax: 512 + NetworkAccess: + networkAccess: true DockerRequirement: dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] @@ -52,24 +51,34 @@ inputs: item: type: string inputBinding: - prefix: --input-item + prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. aoi: type: string inputBinding: - prefix: --aoi + prefix: --aoi + label: Area of interest + doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. epsg: type: string inputBinding: - prefix: --epsg + prefix: --epsg + label: Coordinate reference system + doc: EPSG code of the coordinate reference system used for the area of interest. band: type: string inputBinding: - prefix: --band + prefix: --band + label: Raster band + doc: Name of the STAC asset band to crop. outputs: cropped: outputBinding: - glob: '*.tif' + glob: '*.tif' type: File + label: Cropped raster + doc: Raster band cropped to the area of interest. diff --git a/cwl-cli/norm-diff.cwl b/cwl-cli/norm-diff.cwl index 304e48d..5fb2465 100644 --- a/cwl-cli/norm-diff.cwl +++ b/cwl-cli/norm-diff.cwl @@ -3,8 +3,7 @@ $namespaces: s: https://schema.org/ # Training-example authors and creation date mirror codemeta.json. s:name: Normalized difference -s:description: Compute the normalized difference water index from cropped raster - bands. +s:description: Compute the normalized difference water index from cropped raster bands. s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -17,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool @@ -45,7 +44,6 @@ requirements: ramMax: 512 NetworkAccess: networkAccess: false -hints: DockerRequirement: dockerPull: localhost/norm-diff:latest baseCommand: ["python", "-m", "app"] @@ -55,8 +53,12 @@ inputs: type: File[] inputBinding: position: 1 + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: ndwi: outputBinding: glob: '*.tif' - type: File \ No newline at end of file + type: File + label: Normalized difference water index + doc: Water index raster computed from the input bands. diff --git a/cwl-cli/otsu.cwl b/cwl-cli/otsu.cwl index ddbdc4c..e7de87e 100644 --- a/cwl-cli/otsu.cwl +++ b/cwl-cli/otsu.cwl @@ -3,8 +3,7 @@ $namespaces: s: https://schema.org/ # Training-example authors and creation date mirror codemeta.json. s:name: Otsu threshold -s:description: Detect water bodies by applying the Otsu threshold to a water - index raster. +s:description: Detect water bodies by applying the Otsu threshold to a water index raster. s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -17,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool @@ -45,9 +44,8 @@ requirements: ramMax: 512 NetworkAccess: networkAccess: false -hints: DockerRequirement: - dockerPull: localhost/otsu:latest + dockerPull: localhost/otsu:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -55,8 +53,12 @@ inputs: type: File inputBinding: position: 1 + label: Water index raster + doc: Normalized difference water index raster to threshold. outputs: binary_mask_item: outputBinding: glob: '*.tif' - type: File \ No newline at end of file + type: File + label: Water body mask + doc: Binary raster mask identifying detected water bodies. diff --git a/cwl-cli/stac.cwl b/cwl-cli/stac.cwl index bd4712d..f5109cf 100644 --- a/cwl-cli/stac.cwl +++ b/cwl-cli/stac.cwl @@ -16,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool @@ -44,9 +44,8 @@ requirements: ramMax: 512 NetworkAccess: networkAccess: true -hints: DockerRequirement: - dockerPull: localhost/stac:latest + dockerPull: localhost/stac:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -54,12 +53,18 @@ inputs: type: string inputBinding: prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. raster: type: File inputBinding: prefix: --water-body + label: Water index raster + doc: Normalized difference water index raster to threshold. outputs: stac_catalog: outputBinding: glob: . - type: Directory \ No newline at end of file + type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. diff --git a/cwl-cli/stage-in-released.cwl b/cwl-cli/stage-in-released.cwl index ee34f37..13c4bee 100644 --- a/cwl-cli/stage-in-released.cwl +++ b/cwl-cli/stage-in-released.cwl @@ -16,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool @@ -36,16 +36,20 @@ id: main inputs: reference: type: string + label: STAC item reference + doc: URI of the STAC item whose assets are downloaded. outputs: staged: type: Directory outputBinding: glob: . -baseCommand: -- python -- stage.py + label: Staged acquisition + doc: Directory containing the staged STAC catalog, item, and downloaded assets. +baseCommand: + - python + - stage.py arguments: -- $( inputs.reference ) + - $( inputs.reference ) requirements: DockerRequirement: dockerPull: ghcr.io/eoap/mastering-app-package/stage:1.0.0 diff --git a/cwl-cli/stage-in.cwl b/cwl-cli/stage-in.cwl index 38e4482..a101cd7 100644 --- a/cwl-cli/stage-in.cwl +++ b/cwl-cli/stage-in.cwl @@ -16,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool @@ -36,16 +36,20 @@ id: main inputs: reference: type: string + label: STAC item reference + doc: URI of the STAC item whose assets are downloaded. outputs: staged: type: Directory outputBinding: glob: . -baseCommand: -- python -- stage.py + label: Staged acquisition + doc: Directory containing the staged STAC catalog, item, and downloaded assets. +baseCommand: + - python + - stage.py arguments: -- $( inputs.reference ) + - $( inputs.reference ) requirements: DockerRequirement: dockerPull: localhost/stage:latest diff --git a/cwl-cli/stage-out-released.cwl b/cwl-cli/stage-out-released.cwl index ed77552..6d0341d 100644 --- a/cwl-cli/stage-out-released.cwl +++ b/cwl-cli/stage-out-released.cwl @@ -3,8 +3,7 @@ $namespaces: s: https://schema.org/ # Training-example authors and creation date mirror codemeta.json. s:name: Stage-out STAC results (released container) -s:description: Upload the results catalog and its assets to S3-compatible - storage. +s:description: Upload the results catalog and its assets to S3-compatible storage. s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -17,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 @@ -40,23 +39,39 @@ doc: "Stage-out the results to S3" inputs: s3_bucket: type: string + label: S3 bucket + doc: Name of the destination S3 bucket. sub_path: type: string + label: Object prefix + doc: Object key prefix under which results are uploaded. aws_access_key_id: type: string + label: S3 access key ID + doc: Access key ID used to authenticate to the S3-compatible service. aws_secret_access_key: type: string + label: S3 secret access key + doc: Secret access key used to authenticate to the S3-compatible service. region_name: type: string + label: S3 region + doc: Region name used for S3 requests. endpoint_url: type: string + label: S3 endpoint + doc: URL of the S3-compatible service endpoint. stac_catalog: type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. outputs: s3_catalog_output: outputBinding: outputEval: ${ return "s3://" + inputs.s3_bucket + "/" + inputs.sub_path + "/catalog.json"; } type: string + label: Uploaded STAC catalog + doc: S3 URI of the uploaded results catalog. baseCommand: - python - stage.py @@ -164,4 +179,4 @@ requirements: print(f"upload catalog.json to s3://{bucket}/{subfolder}", file=sys.stderr) pystac.write_file(cat, cat.get_self_href()) - print(f"s3://{bucket}/{subfolder}/catalog.json", file=sys.stdout) \ No newline at end of file + print(f"s3://{bucket}/{subfolder}/catalog.json", file=sys.stdout) diff --git a/cwl-cli/stage-out.cwl b/cwl-cli/stage-out.cwl index 9a31ea9..aa5f20a 100644 --- a/cwl-cli/stage-out.cwl +++ b/cwl-cli/stage-out.cwl @@ -3,8 +3,7 @@ $namespaces: s: https://schema.org/ # Training-example authors and creation date mirror codemeta.json. s:name: Stage-out STAC results -s:description: Upload the results catalog and its assets to S3-compatible - storage. +s:description: Upload the results catalog and its assets to S3-compatible storage. s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -17,19 +16,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 @@ -40,23 +39,39 @@ doc: "Stage-out the results to S3" inputs: s3_bucket: type: string + label: S3 bucket + doc: Name of the destination S3 bucket. sub_path: type: string + label: Object prefix + doc: Object key prefix under which results are uploaded. aws_access_key_id: type: string + label: S3 access key ID + doc: Access key ID used to authenticate to the S3-compatible service. aws_secret_access_key: type: string + label: S3 secret access key + doc: Secret access key used to authenticate to the S3-compatible service. region_name: type: string + label: S3 region + doc: Region name used for S3 requests. endpoint_url: type: string + label: S3 endpoint + doc: URL of the S3-compatible service endpoint. stac_catalog: type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. outputs: s3_catalog_output: outputBinding: outputEval: ${ return "s3://" + inputs.s3_bucket + "/" + inputs.sub_path + "/catalog.json"; } type: string + label: Uploaded STAC catalog + doc: S3 URI of the uploaded results catalog. baseCommand: - python - stage.py @@ -164,4 +179,4 @@ requirements: print(f"upload catalog.json to s3://{bucket}/{subfolder}", file=sys.stderr) pystac.write_file(cat, cat.get_self_href()) - print(f"s3://{bucket}/{subfolder}/catalog.json", file=sys.stdout) \ No newline at end of file + print(f"s3://{bucket}/{subfolder}/catalog.json", file=sys.stdout) diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index 0b9e719..b178899 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -4,8 +4,7 @@ $namespaces: s:softwareVersion: 1.4.1 # Training-example authors and creation date mirror codemeta.json. s:name: Water bodies detection based on NDWI and otsu threshold -s:description: Water bodies detection based on NDWI and otsu threshold applied - to Sentinel-2 COG STAC items +s:description: Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -18,19 +17,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: @@ -65,6 +64,8 @@ $graph: outputSource: - node_stac/stac_catalog type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. steps: node_water_bodies: run: "#detect_water_body" @@ -77,6 +78,8 @@ $graph: - detected_water_body scatter: item scatterMethod: dotproduct + label: Process acquisitions + doc: Run water body detection for each input STAC item. node_stac: run: "#stac" in: @@ -85,6 +88,8 @@ $graph: source: node_water_bodies/detected_water_body out: - stac_catalog + label: Catalog results + doc: Create a STAC catalog describing the detected water bodies. - class: Workflow id: detect_water_body label: Water body detection based on NDWI and otsu threshold @@ -95,21 +100,27 @@ $graph: aoi: doc: area of interest as a bounding box type: string + label: Area of interest epsg: doc: EPSG code type: string default: "EPSG:4326" + label: Coordinate reference system bands: doc: bands used for the NDWI type: string[] + label: NDWI bands item: doc: STAC item type: string + label: STAC item outputs: - id: detected_water_body outputSource: - node_otsu/binary_mask_item type: File + label: Detected water body + doc: Binary raster mask produced by the nested water body detection workflow. steps: node_crop: run: "#crop" @@ -122,6 +133,8 @@ $graph: - cropped scatter: band scatterMethod: dotproduct + label: Crop bands + doc: Crop the selected bands to the area of interest. node_normalized_difference: run: "#norm_diff" in: @@ -129,6 +142,8 @@ $graph: source: node_crop/cropped out: - ndwi + label: Compute NDWI + doc: Compute the normalized difference water index from the cropped bands. node_otsu: run: "#otsu" in: @@ -136,6 +151,8 @@ $graph: source: node_normalized_difference/ndwi out: - binary_mask_item + label: Detect water bodies + doc: Apply the Otsu threshold to produce a binary water body mask. - class: CommandLineTool id: crop requirements: @@ -149,7 +166,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] @@ -159,23 +175,33 @@ $graph: type: string inputBinding: prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. aoi: type: string inputBinding: prefix: --aoi + label: Area of interest + doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. epsg: type: string inputBinding: prefix: --epsg + label: Coordinate reference system + doc: EPSG code of the coordinate reference system used for the area of interest. band: type: string inputBinding: prefix: --band + label: Raster band + doc: Name of the STAC asset band to crop. outputs: cropped: outputBinding: glob: '*.tif' type: File + label: Cropped raster + doc: Raster band cropped to the area of interest. - class: CommandLineTool id: norm_diff requirements: @@ -189,7 +215,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/norm-diff:latest baseCommand: ["python", "-m", "app"] @@ -199,11 +224,15 @@ $graph: type: File[] inputBinding: position: 1 + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: ndwi: outputBinding: glob: '*.tif' type: File + label: Normalized difference water index + doc: Water index raster computed from the input bands. - class: CommandLineTool id: otsu requirements: @@ -217,7 +246,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/otsu:latest baseCommand: ["python", "-m", "app"] @@ -227,11 +255,15 @@ $graph: type: File inputBinding: position: 1 + label: Water index raster + doc: Normalized difference water index raster to threshold. outputs: binary_mask_item: outputBinding: glob: '*.tif' type: File + label: Water body mask + doc: Binary raster mask identifying detected water bodies. - class: CommandLineTool id: stac requirements: @@ -245,7 +277,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/stac:latest baseCommand: ["python", "-m", "app"] @@ -257,14 +288,20 @@ $graph: items: string inputBinding: prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. rasters: type: type: array items: File inputBinding: prefix: --water-body + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: stac_catalog: outputBinding: glob: . type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index 04d312e..deaaabe 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -4,8 +4,7 @@ $namespaces: s:softwareVersion: 1.0.0 # Training-example authors and creation date mirror codemeta.json. s:name: Water bodies detection based on NDWI and the otsu threshold -s:description: Water bodies detection based on NDWI and otsu threshold applied - to a single Sentinel-2 COG STAC item +s:description: Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -18,19 +17,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: @@ -64,6 +63,8 @@ $graph: outputSource: - node_stac/stac_catalog type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. steps: node_crop: run: "#crop" @@ -76,6 +77,8 @@ $graph: - cropped scatter: band scatterMethod: dotproduct + label: Crop bands + doc: Crop the selected bands to the area of interest. node_normalized_difference: run: "#norm_diff" in: @@ -83,6 +86,8 @@ $graph: source: node_crop/cropped out: - ndwi + label: Compute NDWI + doc: Compute the normalized difference water index from the cropped bands. node_otsu: run: "#otsu" in: @@ -90,6 +95,8 @@ $graph: source: node_normalized_difference/ndwi out: - binary_mask_item + label: Detect water bodies + doc: Apply the Otsu threshold to produce a binary water body mask. node_stac: run: "#stac" in: @@ -98,6 +105,8 @@ $graph: source: node_otsu/binary_mask_item out: - stac_catalog + label: Catalog results + doc: Create a STAC catalog describing the detected water bodies. - class: CommandLineTool id: crop requirements: @@ -111,7 +120,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] @@ -121,23 +129,33 @@ $graph: type: string inputBinding: prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. aoi: type: string inputBinding: prefix: --aoi + label: Area of interest + doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. epsg: type: string inputBinding: prefix: --epsg + label: Coordinate reference system + doc: EPSG code of the coordinate reference system used for the area of interest. band: type: string inputBinding: prefix: --band + label: Raster band + doc: Name of the STAC asset band to crop. outputs: cropped: outputBinding: glob: '*.tif' type: File + label: Cropped raster + doc: Raster band cropped to the area of interest. - class: CommandLineTool id: norm_diff requirements: @@ -151,7 +169,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/norm-diff:latest baseCommand: ["python", "-m", "app"] @@ -161,11 +178,15 @@ $graph: type: File[] inputBinding: position: 1 + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: ndwi: outputBinding: glob: '*.tif' type: File + label: Normalized difference water index + doc: Water index raster computed from the input bands. - class: CommandLineTool id: otsu requirements: @@ -179,7 +200,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/otsu:latest baseCommand: ["python", "-m", "app"] @@ -189,11 +209,15 @@ $graph: type: File inputBinding: position: 1 + label: Water index raster + doc: Normalized difference water index raster to threshold. outputs: binary_mask_item: outputBinding: glob: '*.tif' type: File + label: Water body mask + doc: Binary raster mask identifying detected water bodies. - class: CommandLineTool id: stac requirements: @@ -207,7 +231,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/stac:latest baseCommand: ["python", "-m", "app"] @@ -217,12 +240,18 @@ $graph: type: string inputBinding: prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. rasters: type: File inputBinding: prefix: --water-body + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: stac_catalog: outputBinding: glob: . type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index 96f0fce..65b97c9 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -4,8 +4,7 @@ $namespaces: s:softwareVersion: 1.0.0 # Training-example authors and creation date mirror codemeta.json. s:name: Water body detection based on NDWI and the otsu threshold -s:description: Water bodies detection based on NDWI and otsu threshold applied - to Sentinel-2 or Landsat-9 staged acquisitions +s:description: Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or Landsat-9 staged acquisitions s:dateCreated: '2022-09-01' s:license: s:name: Creative Commons Attribution-ShareAlike 4.0 International @@ -18,19 +17,19 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf $graph: @@ -64,6 +63,8 @@ $graph: outputSource: - node_stac/stac_catalog type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. steps: node_crop: run: "#crop" @@ -76,6 +77,8 @@ $graph: - cropped scatter: band scatterMethod: dotproduct + label: Crop bands + doc: Crop the selected bands to the area of interest. node_normalized_difference: run: "#norm_diff" in: @@ -83,6 +86,8 @@ $graph: source: node_crop/cropped out: - ndwi + label: Compute NDWI + doc: Compute the normalized difference water index from the cropped bands. node_otsu: run: "#otsu" in: @@ -90,6 +95,8 @@ $graph: source: node_normalized_difference/ndwi out: - binary_mask_item + label: Detect water bodies + doc: Apply the Otsu threshold to produce a binary water body mask. node_stac: run: "#stac" in: @@ -98,6 +105,8 @@ $graph: source: node_otsu/binary_mask_item out: - stac_catalog + label: Catalog results + doc: Create a STAC catalog describing the detected water bodies. - class: CommandLineTool id: crop requirements: @@ -111,7 +120,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] @@ -121,23 +129,33 @@ $graph: type: Directory inputBinding: prefix: --input-item + label: STAC item + doc: Directory containing the staged acquisition STAC catalog and assets. aoi: type: string inputBinding: prefix: --aoi + label: Area of interest + doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. epsg: type: string inputBinding: prefix: --epsg + label: Coordinate reference system + doc: EPSG code of the coordinate reference system used for the area of interest. band: type: string inputBinding: prefix: --band + label: Raster band + doc: Name of the STAC asset band to crop. outputs: cropped: outputBinding: glob: '*.tif' type: File + label: Cropped raster + doc: Raster band cropped to the area of interest. - class: CommandLineTool id: norm_diff requirements: @@ -151,7 +169,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/norm-diff:latest baseCommand: ["python", "-m", "app"] @@ -161,11 +178,15 @@ $graph: type: File[] inputBinding: position: 1 + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: ndwi: outputBinding: glob: '*.tif' type: File + label: Normalized difference water index + doc: Water index raster computed from the input bands. - class: CommandLineTool id: otsu requirements: @@ -179,7 +200,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: false - hints: DockerRequirement: dockerPull: localhost/otsu:latest baseCommand: ["python", "-m", "app"] @@ -189,11 +209,15 @@ $graph: type: File inputBinding: position: 1 + label: Water index raster + doc: Normalized difference water index raster to threshold. outputs: binary_mask_item: outputBinding: glob: '*.tif' type: File + label: Water body mask + doc: Binary raster mask identifying detected water bodies. - class: CommandLineTool id: stac requirements: @@ -207,7 +231,6 @@ $graph: ramMax: 512 NetworkAccess: networkAccess: true - hints: DockerRequirement: dockerPull: localhost/stac:latest baseCommand: ["python", "-m", "app"] @@ -217,12 +240,18 @@ $graph: type: Directory inputBinding: prefix: --input-item + label: STAC item + doc: Directory containing the staged acquisition STAC catalog and assets. rasters: type: File inputBinding: prefix: --water-body + label: Input rasters + doc: Raster files used for normalized difference computation or catalog generation. outputs: stac_catalog: outputBinding: glob: . type: Directory + label: Results STAC catalog + doc: Directory containing the results STAC catalog and its referenced raster assets. diff --git a/docs/crop.cwl b/docs/crop.cwl index 9e2d753..5313a67 100644 --- a/docs/crop.cwl +++ b/docs/crop.cwl @@ -16,59 +16,68 @@ s:publisher: s:name: EO Application Packaging s:identifier: https://github.com/eoap s:author: -- s:givenName: Jane - s:familyName: Doe - s:email: jane.doe@acme.earth - s:affiliation: - s:name: ACME -- s:givenName: John - s:familyName: Doe - s:email: john.doe@acme.earth - s:affiliation: - s:name: ACME + - s:givenName: Jane + s:familyName: Doe + s:email: jane.doe@acme.earth + s:affiliation: + s:name: ACME + - s:givenName: John + s:familyName: Doe + s:email: john.doe@acme.earth + s:affiliation: + s:name: ACME s:softwareRequirements: -- container runtime -- cwl runner + - container runtime + - cwl runner s:softwareVersion: 1.1.0 class: CommandLineTool id: crop requirements: - InlineJavascriptRequirement: {} - EnvVarRequirement: - envDef: - PYTHONPATH: /app - PROJ_LIB: /opt/conda/envs/env_crop/share/proj/ - ResourceRequirement: - coresMax: 1 - ramMax: 512 -hints: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PYTHONPATH: /app + PROJ_LIB: /opt/conda/envs/env_crop/share/proj/ + ResourceRequirement: + coresMax: 1 + ramMax: 512 DockerRequirement: - dockerPull: localhost/crop:latest + dockerPull: localhost/crop:latest baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: type: string inputBinding: - prefix: --input-item + prefix: --input-item + label: STAC item + doc: STAC item reference identifying the source acquisition. aoi: type: string inputBinding: - prefix: --aoi + prefix: --aoi + label: Area of interest + doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. epsg: type: string inputBinding: - prefix: --epsg + prefix: --epsg + label: Coordinate reference system + doc: EPSG code of the coordinate reference system used for the area of interest. band: type: string inputBinding: - prefix: --band + prefix: --band + label: Raster band + doc: Name of the STAC asset band to crop. outputs: cropped: outputBinding: - glob: '*.tif' + glob: '*.tif' type: File + label: Cropped raster + doc: Raster band cropped to the area of interest. diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md index 1b8e7ba..1cff368 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:20:08.682 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:33:15.396 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -57,26 +57,26 @@ The are no Supported Operating Systems specified for this project. | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | bands used for the NDWI | -| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | STAC item | +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Area of interest | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Coordinate reference system | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | NDWI bands | bands used for the NDWI | +| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item | STAC item | ### Steps | Id | Runs | Label | Doc | |----|------|-------|-----| -| [node_crop](#crop) | `#crop` | None | None | -| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | -| [node_otsu](#otsu) | `#otsu` | None | None | +| [node_crop](#crop) | `#crop` | Crop bands | Crop the selected bands to the area of interest. | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | Compute NDWI | Compute the normalized difference water index from the cropped bands. | +| [node_otsu](#otsu) | `#otsu` | Detect water bodies | Apply the Otsu threshold to produce a binary water body mask. | ### Outputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | None | None | +| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | Detected water body | Binary raster mask produced by the nested water body detection workflow. | ### OGC API - Processes diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/activity.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/activity.svg new file mode 100644 index 0000000..3d2cd46 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/activity.svg @@ -0,0 +1 @@ +detect_water_bodyaoiepsgbandsitemstep: node_cropCommandLineTool: cropdotproduct node_crop/bandstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsudetected_water_body \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg new file mode 100644 index 0000000..34b807f --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg @@ -0,0 +1 @@ +Water body detection based on NDWI and otsu thresholdStepsRequirementsdetect_water_body aoi: string epsg: string = EPSG:4326 bands: string[] item: string  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/component.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/component.svg new file mode 100644 index 0000000..867f2f3 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/component.svg @@ -0,0 +1 @@ +Workflow 'detect_water_body'detect_water_bodynode_cropnode_normalized_differencenode_otsuaoiepsgbandsitemdetected_water_bodyitemaoiepsgbandcroppedcroprastersIN_detect_water_body_node_crop_croppedndwinorm_diffrasterIN_detect_water_body_node_normalized_difference_ndwibinary_mask_itemotsu \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg new file mode 100644 index 0000000..9faa179 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg @@ -0,0 +1 @@ +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleArea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleCoordinate reference systemdescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleNDWI bandsdescriptionbands used for the NDWItypearrayitems   typestring   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC itemdescriptionSTAC itemtypestring   titlecwl:typevaluestring \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_outputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_outputs.svg new file mode 100644 index 0000000..bb6b00c --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_outputs.svg @@ -0,0 +1 @@ +detected_water_body   schema   metadata   titleDetected water bodydescriptionBinary raster mask produced by the nested water body detection workflow.oneOf         typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json   titlecwl:typevalueFile \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/sequence.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/sequence.svg new file mode 100644 index 0000000..0b4f70a --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/sequence.svg @@ -0,0 +1 @@ +Workflow runnerdetect_water_bodystep. node_cropstep. node_normalized_differencestep. node_otsuWorkflow runnerdetect_water_bodystep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsuWorkflow runnerdetect_water_bodystep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsuWater body detection based on NDWI and otsu threshold[001]aoi[002]epsg[003]bands[004]itemloop[dotproduct node_crop/band][005]item(fromitem)[006]aoi(fromaoi)[007]epsg(fromepsg)[008]band(frombands)[009]Compute[010]cropped[011]rasters(fromnode_crop/cropped)[012]Compute[013]ndwi[014]raster(fromnode_normalized_difference/ndwi)[015]Compute[016]binary_mask_item[017]detected_water_body(fromnode_otsu/binary_mask_item) \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/state.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/state.svg new file mode 100644 index 0000000..1b129e9 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/state.svg @@ -0,0 +1 @@ +detect_water_bodynode_cropnode_normalized_differencenode_otsuaoiepsgbandsitemdetected_water_bodyitemaoiepsgbandcroppedrastersndwirasterbinary_mask_item \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md index 0df6393..5bc960e 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:20:08.520 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:33:15.457 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -68,15 +68,15 @@ The are no Supported Operating Systems specified for this project. | Id | Runs | Label | Doc | |----|------|-------|-----| -| [node_water_bodies](#detect_water_body) | `#detect_water_body` | None | None | -| [node_stac](#stac) | `#stac` | None | None | +| [node_water_bodies](#detect_water_body) | `#detect_water_body` | Process acquisitions | Run water body detection for each input STAC item. | +| [node_stac](#stac) | `#stac` | Catalog results | Create a STAC catalog describing the detected water bodies. | ### Outputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | Results STAC catalog | Directory containing the results STAC catalog and its referenced raster assets. | ### OGC API - Processes @@ -148,26 +148,26 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | bands used for the NDWI | -| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | None | STAC item | +| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Area of interest | area of interest as a bounding box | +| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Coordinate reference system | EPSG code | +| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | NDWI bands | bands used for the NDWI | +| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item | STAC item | ### Steps | Id | Runs | Label | Doc | |----|------|-------|-----| -| [node_crop](#crop) | `#crop` | None | None | -| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | -| [node_otsu](#otsu) | `#otsu` | None | None | +| [node_crop](#crop) | `#crop` | Crop bands | Crop the selected bands to the area of interest. | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | Compute NDWI | Compute the normalized difference water index from the cropped bands. | +| [node_otsu](#otsu) | `#otsu` | Detect water bodies | Apply the Otsu threshold to produce a binary water body mask. | ### Outputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | None | None | +| `detected_water_body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | Detected water body | Binary raster mask produced by the nested water body detection workflow. | ### OGC API - Processes diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/activity.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/activity.svg new file mode 100644 index 0000000..b55492d --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/activity.svg @@ -0,0 +1 @@ +water-bodiesaoiepsgstac_itemsbandsdetect_water_bodystep: node_cropCommandLineTool: cropdotproduct node_crop/bandstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsudotproduct node_water_bodies/itemstep: node_stacCommandLineTool: stacstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg new file mode 100644 index 0000000..bcae5bd --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg @@ -0,0 +1 @@ +Water bodies detection based on NDWI and otsu thresholdStepsWater body detection based on NDWI and otsu thresholdStepsRequirementsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 stac_items: string[] bands: string[] = ['green', 'nir']  Inputs stac_catalog: Directory   Outputs node_water_bodies: detect_water_body node_stac: stac  StepsWorkflowstac item: string[] rasters: File[]  Inputs stac_catalog: Directory   OutputsCommandLineTooldetect_water_body aoi: string epsg: string = EPSG:4326 bands: string[] item: string  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirementScatterFeatureRequirementSubworkflowFeatureRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/component.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/component.svg new file mode 100644 index 0000000..9611d9d --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/component.svg @@ -0,0 +1 @@ +Workflow 'water-bodies'water-bodiesnode_water_bodiesnode_stacaoiepsgstac_itemsbandsstac_catalogitemaoiepsgbandsdetected_water_bodydetect_water_bodyitemrastersIN_water_bodies_node_water_bodies_detected_water_bodystac_catalogstac \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg new file mode 100644 index 0000000..3e8d965 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -0,0 +1 @@ +aoi   epsg   stac_items   bands   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSentinel-2 STAC itemsdescriptionlist of Sentinel-2 COG STAC itemstypearrayitems   typestring   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[] \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_outputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_outputs.svg new file mode 100644 index 0000000..ffb6379 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_outputs.svg @@ -0,0 +1 @@ +stac_catalog   schema   metadata   titleResults STAC catalogdescriptionDirectory containing the results STAC catalog and its referenced raster assets.oneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/sequence.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/sequence.svg new file mode 100644 index 0000000..b88dcc5 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/sequence.svg @@ -0,0 +1 @@ +Water body detection based on NDWI and otsu thresholdWorkflow runnerwater-bodiesstep. node_water_bodiesstep. node_cropstep. node_normalized_differencestep. node_otsustep. node_stacWorkflow runnerwater-bodiesstep: node_water_bodiesWorkflow: detect_water_bodystep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWorkflow runnerwater-bodiesstep: node_water_bodiesWorkflow: detect_water_bodystep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWater bodies detection based on NDWI and otsu threshold[001]aoi[002]epsg[003]stac_items[004]bandsloop[dotproduct node_water_bodies/item][005]item(fromstac_items)[006]aoi(fromaoi)[007]epsg(fromepsg)[008]bands(frombands)loop[dotproduct node_crop/band][009]item(fromitem)[010]aoi(fromaoi)[011]epsg(fromepsg)[012]band(frombands)[013]Compute[014]cropped[015]rasters(fromnode_crop/cropped)[016]Compute[017]ndwi[018]raster(fromnode_normalized_difference/ndwi)[019]Compute[020]binary_mask_item[021]detected_water_body(fromnode_otsu/binary_mask_item)[022]item(fromstac_items)[023]rasters(fromnode_water_bodies/detected_water_body)[024]Compute[025]stac_catalog[026]stac_catalog(fromnode_stac/stac_catalog) \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/state.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/state.svg new file mode 100644 index 0000000..429b54e --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/state.svg @@ -0,0 +1 @@ +water-bodieswater-bodiesnode_cropnode_normalized_differencenode_otsunode_stacaoiepsgstac_itemsbandsstac_catalogaoiepsgbandsitemdetected_water_bodyitemaoiepsgbandcroppedrastersndwirasterbinary_mask_itemitemrastersstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md index 78303d2..15fbf66 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to a single Sent > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:20:11.475 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:33:17.352 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -67,17 +67,17 @@ The are no Supported Operating Systems specified for this project. | Id | Runs | Label | Doc | |----|------|-------|-----| -| [node_crop](#crop) | `#crop` | None | None | -| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | -| [node_otsu](#otsu) | `#otsu` | None | None | -| [node_stac](#stac) | `#stac` | None | None | +| [node_crop](#crop) | `#crop` | Crop bands | Crop the selected bands to the area of interest. | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | Compute NDWI | Compute the normalized difference water index from the cropped bands. | +| [node_otsu](#otsu) | `#otsu` | Detect water bodies | Apply the Otsu threshold to produce a binary water body mask. | +| [node_stac](#stac) | `#stac` | Catalog results | Create a STAC catalog describing the detected water bodies. | ### Outputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | Results STAC catalog | Directory containing the results STAC catalog and its referenced raster assets. | ### OGC API - Processes diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/activity.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/activity.svg new file mode 100644 index 0000000..05d1039 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/activity.svg @@ -0,0 +1 @@ +water-bodiesaoiepsgbandsitemstep: node_cropCommandLineTool: cropdotproduct node_crop/bandstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg new file mode 100644 index 0000000..d473ee0 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg @@ -0,0 +1 @@ +Water bodies detection based on NDWI and the otsu thresholdStepsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 bands: string[] = ['green', 'nir'] item: string  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: string rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/component.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/component.svg new file mode 100644 index 0000000..229a105 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/component.svg @@ -0,0 +1 @@ +Workflow 'water-bodies'water-bodiesnode_cropnode_normalized_differencenode_otsunode_stacaoiepsgbandsitemstac_catalogitemaoiepsgbandcroppedcroprastersIN_water_bodies_node_crop_croppedndwinorm_diffrasterIN_water_bodies_node_normalized_difference_ndwibinary_mask_itemotsuitemrastersIN_water_bodies_node_otsu_binary_mask_itemstac_catalogstac \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg new file mode 100644 index 0000000..95e58c2 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -0,0 +1 @@ +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemtypestring   titlecwl:typevaluestring \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_outputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_outputs.svg new file mode 100644 index 0000000..ffb6379 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_outputs.svg @@ -0,0 +1 @@ +stac_catalog   schema   metadata   titleResults STAC catalogdescriptionDirectory containing the results STAC catalog and its referenced raster assets.oneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/sequence.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/sequence.svg new file mode 100644 index 0000000..c4b3c6f --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/sequence.svg @@ -0,0 +1 @@ +Workflow runnerwater-bodiesstep. node_cropstep. node_normalized_differencestep. node_otsustep. node_stacWorkflow runnerwater-bodiesstep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWorkflow runnerwater-bodiesstep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWater bodies detection based on NDWI and the otsu threshold[001]aoi[002]epsg[003]bands[004]itemloop[dotproduct node_crop/band][005]item(fromitem)[006]aoi(fromaoi)[007]epsg(fromepsg)[008]band(frombands)[009]Compute[010]cropped[011]rasters(fromnode_crop/cropped)[012]Compute[013]ndwi[014]raster(fromnode_normalized_difference/ndwi)[015]Compute[016]binary_mask_item[017]item(fromitem)[018]rasters(fromnode_otsu/binary_mask_item)[019]Compute[020]stac_catalog[021]stac_catalog(fromnode_stac/stac_catalog) \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/state.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/state.svg new file mode 100644 index 0000000..6f20264 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/state.svg @@ -0,0 +1 @@ +water-bodiesnode_cropnode_normalized_differencenode_otsunode_stacaoiepsgbandsitemstac_catalogitemaoiepsgbandcroppedrastersndwirasterbinary_mask_itemitemrastersstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md index d45b113..8b729d7 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:20:13.780 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:33:19.131 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -67,17 +67,17 @@ The are no Supported Operating Systems specified for this project. | Id | Runs | Label | Doc | |----|------|-------|-----| -| [node_crop](#crop) | `#crop` | None | None | -| [node_normalized_difference](#norm_diff) | `#norm_diff` | None | None | -| [node_otsu](#otsu) | `#otsu` | None | None | -| [node_stac](#stac) | `#stac` | None | None | +| [node_crop](#crop) | `#crop` | Crop bands | Crop the selected bands to the area of interest. | +| [node_normalized_difference](#norm_diff) | `#norm_diff` | Compute NDWI | Compute the normalized difference water index from the cropped bands. | +| [node_otsu](#otsu) | `#otsu` | Detect water bodies | Apply the Otsu threshold to produce a binary water body mask. | +| [node_stac](#stac) | `#stac` | Catalog results | Create a STAC catalog describing the detected water bodies. | ### Outputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | None | None | +| `stac_catalog` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | Results STAC catalog | Directory containing the results STAC catalog and its referenced raster assets. | ### OGC API - Processes diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/activity.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/activity.svg new file mode 100644 index 0000000..05d1039 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/activity.svg @@ -0,0 +1 @@ +water-bodiesaoiepsgbandsitemstep: node_cropCommandLineTool: cropdotproduct node_crop/bandstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg new file mode 100644 index 0000000..17bbc15 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg @@ -0,0 +1 @@ +Water body detection based on NDWI and the otsu thresholdStepsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 bands: string[] = ['green', 'nir'] item: Directory  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: Directory aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: Directory rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/component.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/component.svg new file mode 100644 index 0000000..229a105 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/component.svg @@ -0,0 +1 @@ +Workflow 'water-bodies'water-bodiesnode_cropnode_normalized_differencenode_otsunode_stacaoiepsgbandsitemstac_catalogitemaoiepsgbandcroppedcroprastersIN_water_bodies_node_crop_croppedndwinorm_diffrasterIN_water_bodies_node_normalized_difference_ndwibinary_mask_itemotsuitemrastersIN_water_bodies_node_otsu_binary_mask_itemstac_catalogstac \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg new file mode 100644 index 0000000..1a3d552 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg @@ -0,0 +1 @@ +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemoneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_outputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_outputs.svg new file mode 100644 index 0000000..ffb6379 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_outputs.svg @@ -0,0 +1 @@ +stac_catalog   schema   metadata   titleResults STAC catalogdescriptionDirectory containing the results STAC catalog and its referenced raster assets.oneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/sequence.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/sequence.svg new file mode 100644 index 0000000..095b5ac --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/sequence.svg @@ -0,0 +1 @@ +Workflow runnerwater-bodiesstep. node_cropstep. node_normalized_differencestep. node_otsustep. node_stacWorkflow runnerwater-bodiesstep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWorkflow runnerwater-bodiesstep: node_cropCommandLineTool: cropstep: node_normalized_differenceCommandLineTool: norm_diffstep: node_otsuCommandLineTool: otsustep: node_stacCommandLineTool: stacWater body detection based on NDWI and the otsu threshold[001]aoi[002]epsg[003]bands[004]itemloop[dotproduct node_crop/band][005]item(fromitem)[006]aoi(fromaoi)[007]epsg(fromepsg)[008]band(frombands)[009]Compute[010]cropped[011]rasters(fromnode_crop/cropped)[012]Compute[013]ndwi[014]raster(fromnode_normalized_difference/ndwi)[015]Compute[016]binary_mask_item[017]item(fromitem)[018]rasters(fromnode_otsu/binary_mask_item)[019]Compute[020]stac_catalog[021]stac_catalog(fromnode_stac/stac_catalog) \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/state.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/state.svg new file mode 100644 index 0000000..6f20264 --- /dev/null +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/state.svg @@ -0,0 +1 @@ +water-bodiesnode_cropnode_normalized_differencenode_otsunode_stacaoiepsgbandsitemstac_catalogitemaoiepsgbandcroppedrastersndwirasterbinary_mask_itemitemrastersstac_catalog \ No newline at end of file diff --git a/docs/reference/cwl/index.md b/docs/reference/cwl/index.md index 28b2f02..a726cd6 100644 --- a/docs/reference/cwl/index.md +++ b/docs/reference/cwl/index.md @@ -41,5 +41,37 @@ preserved; standalone tools use CodeMeta's application version. The generated pages retain upstream template output, including generation timestamps, links for optional fields rendered as `None`, and references to UML -and OGC API schema images. cwl2markdown does not generate those images; they -require separate transpilers and renderers. +and OGC API schema images. The corresponding images are generated separately +with cwl2puml and rendered locally with PlantUML. + +## EOAP validation + +Validate both `water-bodies` and `detect_water_body` in the multi-workflow +document explicitly using their fragments. Package, metadata, and staging +profiles use these reviewed applicability configurations: + +| Entrypoint | Staging configuration | +| --- | --- | +| `app-water-body.cwl` | `validation/staging-staged.json` | +| `app-water-body-cloud-native.cwl` | `validation/staging-cloud-native.json` | +| `app-water-bodies-cloud-native.cwl#water-bodies` | `validation/staging-cloud-native.json` | +| `app-water-bodies-cloud-native.cwl#detect_water_body` | `validation/staging-detection.json` | + +The staged workflow accepts acquisition directories at its entrypoint and in +the crop and catalog tools. Cloud-native inputs are remote STAC references; +their internal raster files are intermediate products, not staged acquisition +directories. The three top-level packages expose a Directory-based STAC catalog +produced by the catalog tool. The nested detection workflow returns an +intermediate raster file and declares no package staging boundary. + +The validator always flags `EOAP.REQ14.COVERAGE` for declared staged outputs: +execution evidence is needed to prove that all produced EO files are collected. +These review findings remain even when Directory type checks pass. + +Standalone command-line tools have no package Workflow entrypoint. Use +`cwltool --validate ` for their CWL validity; their metadata is also +checked by the EOAP validator, but package and staging checks are blocked by +its workflow-selection requirement. + +Container declarations now use mandatory `DockerRequirement` entries rather +than advisory hints, so runners must honor the specified images. diff --git a/validation/staging-cloud-native.json b/validation/staging-cloud-native.json new file mode 100644 index 0000000..6c14570 --- /dev/null +++ b/validation/staging-cloud-native.json @@ -0,0 +1,7 @@ +{ + "inputs": {}, + "outputs": { + "water-bodies": ["stac_catalog"], + "stac": ["stac_catalog"] + } +} diff --git a/validation/staging-detection.json b/validation/staging-detection.json new file mode 100644 index 0000000..1d6a5dd --- /dev/null +++ b/validation/staging-detection.json @@ -0,0 +1,4 @@ +{ + "inputs": {}, + "outputs": {} +} diff --git a/validation/staging-staged.json b/validation/staging-staged.json new file mode 100644 index 0000000..9061ef2 --- /dev/null +++ b/validation/staging-staged.json @@ -0,0 +1,11 @@ +{ + "inputs": { + "water-bodies": ["item"], + "crop": ["item"], + "stac": ["item"] + }, + "outputs": { + "water-bodies": ["stac_catalog"], + "stac": ["stac_catalog"] + } +} From a1b3f32e243cf51de0face46dacf19af57c8ac82 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:52:07 +0200 Subject: [PATCH 05/23] feat(cwl)!: adopt typed EOAP inputs and shared enum schemas Import GeoJSON Polygon and URI records, share EPSG and spectral-band enums, and adapt command bindings to the existing application interface. BREAKING CHANGE: Local CWL jobs must supply Polygon AOIs and URI records instead of plain strings. Workflow EPSG values now use the 4326 enum. --- cwl-cli/crop-green-params.yaml | 22 +++++++ cwl-cli/crop-params.yaml | 26 ++++++-- cwl-cli/crop.cwl | 12 +++- .../app-water-bodies-cloud-native.cwl | 64 +++++++++++++------ cwl-workflow/app-water-body-cloud-native.cwl | 43 +++++++++---- cwl-workflow/app-water-body.cwl | 30 ++++++--- cwl-workflow/eoap-types.yaml | 6 ++ cwl-workflow/typed-cloud-native-inputs.yaml | 13 ++++ cwl-workflow/typed-scatter-inputs.yaml | 25 ++++++++ docs/crop.cwl | 12 +++- 10 files changed, 204 insertions(+), 49 deletions(-) create mode 100644 cwl-cli/crop-green-params.yaml create mode 100644 cwl-workflow/eoap-types.yaml create mode 100644 cwl-workflow/typed-cloud-native-inputs.yaml create mode 100644 cwl-workflow/typed-scatter-inputs.yaml diff --git a/cwl-cli/crop-green-params.yaml b/cwl-cli/crop-green-params.yaml new file mode 100644 index 0000000..a19b82d --- /dev/null +++ b/cwl-cli/crop-green-params.yaml @@ -0,0 +1,22 @@ +item: + value: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A +aoi: + type: Polygon + coordinates: + - - - -121.399 + - 39.834 + - - -120.74 + - 39.834 + - - -120.74 + - 40.472 + - - -121.399 + - 40.472 + - - -121.399 + - 39.834 + bbox: + - -121.399 + - 39.834 + - -120.74 + - 40.472 +epsg: EPSG:4326 +band: green diff --git a/cwl-cli/crop-params.yaml b/cwl-cli/crop-params.yaml index b96880e..d44637f 100644 --- a/cwl-cli/crop-params.yaml +++ b/cwl-cli/crop-params.yaml @@ -1,4 +1,22 @@ -item: "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" -aoi: "-121.399,39.834,-120.74,40.472" -epsg: "EPSG:4326" -band: "green" \ No newline at end of file +item: + value: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A +aoi: + type: Polygon + coordinates: + - - - -121.399 + - 39.834 + - - -120.74 + - 39.834 + - - -120.74 + - 40.472 + - - -121.399 + - 40.472 + - - -121.399 + - 39.834 + bbox: + - -121.399 + - 39.834 + - -120.74 + - 40.472 +epsg: EPSG:4326 +band: nir diff --git a/cwl-cli/crop.cwl b/cwl-cli/crop.cwl index 8cead2e..74f2b67 100644 --- a/cwl-cli/crop.cwl +++ b/cwl-cli/crop.cwl @@ -45,21 +45,27 @@ requirements: networkAccess: true DockerRequirement: dockerPull: localhost/crop:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. aoi: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi + valueFrom: $(self.bbox.join(",")) label: Area of interest - doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. epsg: type: string inputBinding: diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index b178899..12aad03 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -40,24 +40,33 @@ $graph: requirements: - class: ScatterFeatureRequirement - class: SubworkflowFeatureRequirement + - class: SchemaDefRequirement + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml + - $import: eoap-types.yaml inputs: aoi: label: area of interest - doc: area of interest as a bounding box - type: string + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code doc: EPSG code - type: string - default: "EPSG:4326" + type: 'eoap-types.yaml#EPSGCode' + default: "4326" stac_items: label: Sentinel-2 STAC items doc: list of Sentinel-2 COG STAC items - type: string[] + type: + type: array + items: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI bands: label: bands used for the NDWI doc: bands used for the NDWI - type: string[] + type: + type: array + items: 'eoap-types.yaml#SpectralBand' default: ["green", "nir"] outputs: - id: stac_catalog @@ -96,23 +105,30 @@ $graph: doc: Water body detection based on NDWI and otsu threshold requirements: - class: ScatterFeatureRequirement + - class: SchemaDefRequirement + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml + - $import: eoap-types.yaml inputs: aoi: - doc: area of interest as a bounding box - type: string + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon label: Area of interest epsg: doc: EPSG code - type: string - default: "EPSG:4326" + type: 'eoap-types.yaml#EPSGCode' + default: "4326" label: Coordinate reference system bands: doc: bands used for the NDWI - type: string[] + type: + type: array + items: 'eoap-types.yaml#SpectralBand' label: NDWI bands item: doc: STAC item - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI label: STAC item outputs: - id: detected_water_body @@ -168,29 +184,37 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/crop:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml + - $import: eoap-types.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. aoi: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi + valueFrom: $(self.bbox.join(",")) label: Area of interest - doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. epsg: - type: string + type: 'eoap-types.yaml#EPSGCode' inputBinding: prefix: --epsg + valueFrom: $(self.split(":").pop()) label: Coordinate reference system doc: EPSG code of the coordinate reference system used for the area of interest. band: - type: string + type: 'eoap-types.yaml#SpectralBand' inputBinding: prefix: --band label: Raster band @@ -279,15 +303,19 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/stac:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: type: type: array - items: string + items: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. rasters: diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index deaaabe..f790eaf 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -39,25 +39,32 @@ $graph: doc: Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item requirements: - class: ScatterFeatureRequirement + - class: SchemaDefRequirement + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml + - $import: eoap-types.yaml inputs: aoi: label: area of interest - doc: area of interest as a bounding box - type: string + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code doc: EPSG code - type: string - default: "EPSG:4326" + type: 'eoap-types.yaml#EPSGCode' + default: "4326" bands: label: bands used for the NDWI doc: bands used for the NDWI - type: string[] + type: + type: array + items: 'eoap-types.yaml#SpectralBand' default: ["green", "nir"] item: doc: Reference to a STAC item label: STAC item reference - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI outputs: - id: stac_catalog outputSource: @@ -122,29 +129,37 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/crop:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml + - $import: eoap-types.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. aoi: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi + valueFrom: $(self.bbox.join(",")) label: Area of interest - doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. epsg: - type: string + type: 'eoap-types.yaml#EPSGCode' inputBinding: prefix: --epsg + valueFrom: $(self.split(":").pop()) label: Coordinate reference system doc: EPSG code of the coordinate reference system used for the area of interest. band: - type: string + type: 'eoap-types.yaml#SpectralBand' inputBinding: prefix: --band label: Raster band @@ -233,13 +248,17 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/stac:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. rasters: diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index 65b97c9..f45b3bd 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -39,20 +39,26 @@ $graph: doc: Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or Landsat-9 staged acquisitions requirements: - class: ScatterFeatureRequirement + - class: SchemaDefRequirement + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: eoap-types.yaml inputs: aoi: label: area of interest - doc: area of interest as a bounding box - type: string + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code doc: EPSG code - type: string - default: "EPSG:4326" + type: 'eoap-types.yaml#EPSGCode' + default: "4326" bands: label: bands used for the NDWI doc: bands used for the NDWI - type: string[] + type: + type: array + items: 'eoap-types.yaml#SpectralBand' default: ["green", "nir"] item: doc: Reference to a STAC item @@ -122,6 +128,10 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/crop:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: eoap-types.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: @@ -132,19 +142,21 @@ $graph: label: STAC item doc: Directory containing the staged acquisition STAC catalog and assets. aoi: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi + valueFrom: $(self.bbox.join(",")) label: Area of interest - doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. epsg: - type: string + type: 'eoap-types.yaml#EPSGCode' inputBinding: prefix: --epsg + valueFrom: $(self.split(":").pop()) label: Coordinate reference system doc: EPSG code of the coordinate reference system used for the area of interest. band: - type: string + type: 'eoap-types.yaml#SpectralBand' inputBinding: prefix: --band label: Raster band diff --git a/cwl-workflow/eoap-types.yaml b/cwl-workflow/eoap-types.yaml new file mode 100644 index 0000000..043deaa --- /dev/null +++ b/cwl-workflow/eoap-types.yaml @@ -0,0 +1,6 @@ +- name: EPSGCode + type: enum + symbols: ["4326"] +- name: SpectralBand + type: enum + symbols: [green, nir, nir08] diff --git a/cwl-workflow/typed-cloud-native-inputs.yaml b/cwl-workflow/typed-cloud-native-inputs.yaml new file mode 100644 index 0000000..7c18378 --- /dev/null +++ b/cwl-workflow/typed-cloud-native-inputs.yaml @@ -0,0 +1,13 @@ +item: + value: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A +aoi: + type: Polygon + coordinates: + - - [-121.399, 39.834] + - [-120.74, 39.834] + - [-120.74, 40.472] + - [-121.399, 40.472] + - [-121.399, 39.834] + bbox: [-121.399, 39.834, -120.74, 40.472] +epsg: "4326" +bands: [green, nir] diff --git a/cwl-workflow/typed-scatter-inputs.yaml b/cwl-workflow/typed-scatter-inputs.yaml new file mode 100644 index 0000000..461b557 --- /dev/null +++ b/cwl-workflow/typed-scatter-inputs.yaml @@ -0,0 +1,25 @@ +aoi: + type: Polygon + coordinates: + - - - -121.399 + - 39.834 + - - -120.74 + - 39.834 + - - -120.74 + - 40.472 + - - -121.399 + - 40.472 + - - -121.399 + - 39.834 + bbox: + - -121.399 + - 39.834 + - -120.74 + - 40.472 +epsg: '4326' +bands: + - green + - nir +stac_items: + - value: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A + - value: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210723_0_L2A diff --git a/docs/crop.cwl b/docs/crop.cwl index 5313a67..8502d0f 100644 --- a/docs/crop.cwl +++ b/docs/crop.cwl @@ -44,21 +44,27 @@ requirements: ramMax: 512 DockerRequirement: dockerPull: localhost/crop:latest + SchemaDefRequirement: + types: + - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml + - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml baseCommand: ["python", "-m", "app"] arguments: [] inputs: item: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: prefix: --input-item + valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. aoi: - type: string + type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi + valueFrom: $(self.bbox.join(",")) label: Area of interest - doc: Bounding box delimiting the area to process, expressed in the specified coordinate reference system. + doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. epsg: type: string inputBinding: From 3292d39e8f36c313edaee5cec0d68c4414a47c9d Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 18:52:10 +0200 Subject: [PATCH 06/23] docs(labs): align notebooks and references with typed CWL inputs Use typed YAML jobs in local CWL labs and documentation scripts, document release input compatibility, and refresh generated reference pages and diagrams. --- docs/cwl-cli/crop.md | 16 +++-- docs/cwl-workflow/cloud-native.md | 19 ++++-- docs/cwl-workflow/exec-cloud-native.md | 5 +- docs/cwl-workflow/exec-stage-in.md | 6 +- docs/cwl-workflow/scatter-cloud-native.md | 34 ++++++++-- docs/cwl-workflow/staged.md | 19 +++++- .../detect_water_body.md | 19 +++--- .../detect_water_body/class.svg | 2 +- .../ogc_processes_inputs.svg | 2 +- .../water-bodies.md | 30 +++++---- .../water-bodies/class.svg | 2 +- .../water-bodies/ogc_processes_inputs.svg | 2 +- .../water-bodies.md | 21 +++--- .../water-bodies/class.svg | 2 +- .../water-bodies/ogc_processes_inputs.svg | 2 +- .../app-water-body/water-bodies.md | 15 +++-- .../app-water-body/water-bodies/class.svg | 2 +- .../water-bodies/ogc_processes_inputs.svg | 2 +- docs/reference/cwl/index.md | 26 +++++++ .../3-CWL-CommandLineTools/crop.ipynb | 67 ++++++++++++++++--- .../4-CWL-Workflows/1-cloud-native.ipynb | 26 ++++--- .../4-CWL-Workflows/2.2-staged.ipynb | 45 +++++++++++-- .../3-cloud-native-scatter.ipynb | 27 ++++---- scripts/cwl-cli-crop-green.sh | 15 ++--- scripts/cwl-cli-crop-nir.sh | 15 ++--- scripts/cwl-workflow-cloud-native-scatter.sh | 12 ++-- scripts/cwl-workflow-staged.sh | 38 +++++++++-- 27 files changed, 325 insertions(+), 146 deletions(-) diff --git a/docs/cwl-cli/crop.md b/docs/cwl-cli/crop.md index ceb2752..77822d3 100644 --- a/docs/cwl-cli/crop.md +++ b/docs/cwl-cli/crop.md @@ -5,13 +5,13 @@ Wrap the `crop` step as a Common Workflow Language CommandLineTool and execute i ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/CommandLineTools/crop.ipynb +This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/3-CWL-CommandLineTools/crop.ipynb ### How to wrap a step as a CWL CommandLineTool The CWL document below shows the `crop` step wrapped as a CWL CommandLineTool: -```yaml linenums="1" hl_lines="9-12 49-53" title="cwl-cli/crop.cwl" +```yaml linenums="1" title="cwl-cli/crop.cwl" --8<-- cwl-cli/crop.cwl --8<-- @@ -30,8 +30,8 @@ Let's break down the key components of this CWL document: * `baseCommand`: Defines the base command to be executed in the container. In this case, it's running a Python module called "app" with the command `python -m app`. * `arguments`: This section is empty, meaning there are no additional command-line arguments specified here. The tool is expected to receive its arguments via the input parameters. * `inputs`: Describes the input parameters for the tool, including their types and how they are bound to command-line arguments. The tool expects the following inputs: - * `item`: A string representing the input STAC item (image) to be processed, bound to the `--input-item` argument. - * `aoi`: A string representing the area of interest (AOI) as a bounding box, bound to the `--aoi` argument. + * `item`: A URI record containing a `value` field with the STAC item URL; `valueFrom: $(self.value)` passes that URL, bound to the `--input-item` argument. + * `aoi`: A GeoJSON Polygon record with `coordinates` and `bbox`; `valueFrom: $(self.bbox.join(","))` converts its bounding box to the application argument, bound to the `--aoi` argument. * `epsg`: A string representing the EPSG code for the coordinate system, bound to the `--epsg` argument. * `band`: A string representing the name of the band to be extracted, bound to the `--band` argument. * `outputs`: Specifies the tool's output. It defines an output parameter named `cropped`, which is of type `File`. The outputBinding section specifies that the tool is expected to produce one or more TIFF files (glob: '*.tif') as output. @@ -98,6 +98,8 @@ and run it with: cwltool \ --podman \ --outdir /workspace/mastering-app-package/runs \ - crop.cwl \ - crop-params.yaml -``` \ No newline at end of file + ${WORKSPACE}/cwl-cli/crop.cwl \ + ${WORKSPACE}/cwl-cli/crop-params.yaml +``` + +`SchemaDefRequirement` imports the EOAP GeoJSON and URI record schemas. Use the YAML job above to supply structured inputs. The standalone crop tool retains string `epsg` and `band` inputs; the workflows constrain these through enums. The application still crops a rectangle using `bbox`. diff --git a/docs/cwl-workflow/cloud-native.md b/docs/cwl-workflow/cloud-native.md index 56d1722..d7b5d78 100644 --- a/docs/cwl-workflow/cloud-native.md +++ b/docs/cwl-workflow/cloud-native.md @@ -1,9 +1,9 @@ The Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a single STAC item as input parameters: -* a SpatioTemporal Asset Catalog (STAC) Item -* a bounding box area of interest (AOI) -* the EPSG code of the bounding box area of interest +* a STAC item URI record with a `value` field +* a GeoJSON Polygon area of interest (AOI), with `coordinates` and `bbox` +* the EPSG enum value `"4326"` * a list of common band names (["green", "nir"]) ``` mermaid @@ -35,8 +35,19 @@ STAC Catalog`")) The CWL Workflow is shown below and the lines highlighted chain the steps: -```yaml linenums="1" hl_lines="8-71" title="app-water-body-cloud-native.cwl" +```yaml linenums="1" title="app-water-body-cloud-native.cwl" --8<-- cwl-workflow/app-water-body-cloud-native.cwl --8<-- ``` + + +### Typed inputs + +Use a YAML job document for the local CWL definitions. AOIs require a Polygon record with `type`, `coordinates`, and `bbox`; the crop command uses the supplied bounding box. STAC references use URI records with a `value` field. Workflow EPSG values are `"4326"`, and supported bands are `green`, `nir`, and `nir08`. Staged input acquisitions retain their `Directory` type. + +```yaml +--8<-- +cwl-workflow/typed-cloud-native-inputs.yaml +--8<-- +``` diff --git a/docs/cwl-workflow/exec-cloud-native.md b/docs/cwl-workflow/exec-cloud-native.md index 7aa79a7..0ca1a76 100644 --- a/docs/cwl-workflow/exec-cloud-native.md +++ b/docs/cwl-workflow/exec-cloud-native.md @@ -4,7 +4,7 @@ Run the `app-water-body-cloud-native.1.0.0.cwl` released application package usi ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Workflows/cloud-native.ipynb +This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb ### Step 1 - Configure the workspace @@ -60,3 +60,6 @@ The folder `/workspace/mastering-app-package/runs` contains: 2 directories, 4 files ``` + + +These commands target downloaded release packages and retain those releases' input format. For local typed CWL files, use the YAML job examples in the workflow authoring labs. diff --git a/docs/cwl-workflow/exec-stage-in.md b/docs/cwl-workflow/exec-stage-in.md index c1113d4..6c1e501 100644 --- a/docs/cwl-workflow/exec-stage-in.md +++ b/docs/cwl-workflow/exec-stage-in.md @@ -4,7 +4,7 @@ Run the `app-water-body.1.0.0.cwl` released application package using `cwltool` ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Workflows/staged.ipynb +This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/4-CWL-Workflows/2.2-staged.ipynb ### Step 1 - Configure the workspace @@ -184,4 +184,6 @@ and (base) jovyan@coder-mrossi:~/runs$ aws s3 ls $( cat staged-s3.json | jq -r .s3_catalog_output | xargs dirname )/ PRE LC09_L2SP_042033_20231015_02_T1/ 2023-11-02 08:46:04 549 catalog.json -``` \ No newline at end of file +``` + +These commands target downloaded release packages and retain those releases' input format. For local typed CWL files, use the YAML job examples in the workflow authoring labs. diff --git a/docs/cwl-workflow/scatter-cloud-native.md b/docs/cwl-workflow/scatter-cloud-native.md index edb61ea..3d77c90 100644 --- a/docs/cwl-workflow/scatter-cloud-native.md +++ b/docs/cwl-workflow/scatter-cloud-native.md @@ -34,18 +34,28 @@ STAC Catalog`")) Below a CWL Workflow implementing this scenario: -```yaml linenums="1" hl_lines="18-21 33-43" +```yaml linenums="1" --8<-- -cwl-workflow/app-water-bodies-cloud-native.cwl:8:58 +cwl-workflow/app-water-bodies-cloud-native.cwl --8<-- ``` The `stac` CommandLineTool is updated to manage arrays: -```yaml linenums="200" hl_lines="19-23 25-29" ---8<-- -cwl-workflow/app-water-bodies-cloud-native.cwl:200:234 ---8<-- +```yaml +item: + type: + type: array + items: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI + inputBinding: + prefix: --input-item + valueFrom: $(self.value) +rasters: + type: + type: array + items: File + inputBinding: + prefix: --water-body ``` To run this CWL document, one would do: @@ -54,4 +64,14 @@ To run this CWL document, one would do: --8<-- scripts/cwl-workflow-cloud-native-scatter.sh --8<-- -``` \ No newline at end of file +``` + +### Typed inputs + +Use a YAML job document for the local CWL definitions. AOIs require a Polygon record with `type`, `coordinates`, and `bbox`; the crop command uses the supplied bounding box. STAC references use URI records with a `value` field. Workflow EPSG values are `"4326"`, and supported bands are `green`, `nir`, and `nir08`. Staged input acquisitions retain their `Directory` type. + +```yaml +--8<-- +cwl-workflow/typed-scatter-inputs.yaml +--8<-- +``` diff --git a/docs/cwl-workflow/staged.md b/docs/cwl-workflow/staged.md index ad587a6..1853072 100644 --- a/docs/cwl-workflow/staged.md +++ b/docs/cwl-workflow/staged.md @@ -1,8 +1,8 @@ The non-Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a staged EO acquisition as a STAC Catalog in a directory as input parameters: * a local STAC Catalog in a directory -* a bounding box area of interest (AOI) -* the EPSG code of the bounding box area of interest +* a GeoJSON Polygon area of interest (AOI), with `coordinates` and `bbox` +* the EPSG enum value `"4326"` * a list of common band names (["green", "nir08"]) ``` mermaid @@ -34,9 +34,22 @@ STAC Catalog`")) The CWL Workflow is shown below and the lines highlighted show the changes to manage a staged STAC Catalog instead of an URL to a STAC Item: -```yaml linenums="1" hl_lines="32 90 180" title="app-water-body.cwl" +```yaml linenums="1" title="app-water-body.cwl" --8<-- cwl-workflow/app-water-body.cwl --8<-- ``` + + +### Typed inputs + +Use a YAML job document for the local CWL definitions. AOIs require a Polygon record with `type`, `coordinates`, and `bbox`; the crop command uses the supplied bounding box. STAC references use URI records with a `value` field. Workflow EPSG values are `"4326"`, and supported bands are `green`, `nir`, and `nir08`. Staged input acquisitions retain their `Directory` type. + +The staged processing lab (`practice-labs/4-CWL-Workflows/2.2-staged.ipynb`) creates a job using the directory returned in `staged.json`. + +```bash +--8<-- +scripts/cwl-workflow-staged.sh +--8<-- +``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md index 1cff368..3bd098b 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:33:15.396 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:49:46.289 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -52,15 +52,16 @@ The are no Supported Operating Systems specified for this project. ### Requirements * [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) +* [SchemaDefRequirement](https://www.commonwl.org/v1.2/Workflow.html#SchemaDefRequirement) ### Inputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Area of interest | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Coordinate reference system | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | NDWI bands | bands used for the NDWI | -| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item | STAC item | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| Coordinate reference system | EPSG code | +| `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| NDWI bands | bands used for the NDWI | +| `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item | STAC item | ### Steps @@ -144,10 +145,10 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `item` | `--input-item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `aoi` | `--aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `epsg` | `--epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| +| `band` | `--band` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| ### Execution usage example: diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg index 34b807f..c005d2b 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/class.svg @@ -1 +1 @@ -Water body detection based on NDWI and otsu thresholdStepsRequirementsdetect_water_body aoi: string epsg: string = EPSG:4326 bands: string[] item: string  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirement \ No newline at end of file +SchemasWater body detection based on NDWI and otsu thresholdStepsRequirementsPointTypePointPointtype: PointType coordinates: double[] bbox: double[]LineStringTypeLineStringLineStringtype: LineStringType coordinates: double[] bbox: double[]PolygonTypePolygonPolygontype: PolygonType coordinates: double[][][] bbox: double[]MultiPointTypeMultiPointMultiPointtype: MultiPointType coordinates: double[][] bbox: double[]MultiLineStringTypeMultiLineStringMultiLineStringtype: MultiLineStringType coordinates: double[][][] bbox: double[]MultiPolygonTypeMultiPolygonMultiPolygontype: MultiPolygonType coordinates: double[][][][] bbox: double[]FeatureTypeFeatureFeaturetype: FeatureType id: string geometry: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon bbox: double[]GeometryCollectionTypeGeometryCollectionGeometryCollectiontype: GeometryCollectionType geometries: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon[] bbox: double[]Linkhref: string rel: string type: string hreflang: string title: string length: intFeatureCollectionTypeFeatureCollectionFeatureCollectiontype: FeatureCollectionType features: Feature[] links: Link[] timeStamp: string numberMatched: int numberReturned: intDatevalue: stringDateTimevalue: stringDurationvalue: stringEmailvalue: stringHostnamevalue: stringIDNEmailvalue: stringIDNHostnamevalue: stringIPv4value: stringIPv6value: stringIRIvalue: stringIRIReferencevalue: stringPasswordvalue: stringJsonPointervalue: stringRelativeJsonPointervalue: stringUUIDvalue: stringURIvalue: stringURIReferencevalue: stringURITemplatevalue: stringTimevalue: stringPatternvalue: string pattern: stringEPSGCode4326SpectralBandgreen nir nir08detect_water_body aoi: Polygon epsg: EPSGCode = 4326 bands: SpectralBand[] item: URI  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: URI aoi: Polygon epsg: EPSGCode band: SpectralBand  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirementSchemaDefRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg index 9faa179..acee659 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleArea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleCoordinate reference systemdescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleNDWI bandsdescriptionbands used for the NDWItypearrayitems   typestring   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC itemdescriptionSTAC itemtypestring   titlecwl:typevaluestring \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleArea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleCoordinate reference systemdescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleNDWI bandsdescriptionbands used for the NDWItypearrayitems   typestringenum   greennirnir08   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC itemdescriptionSTAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md index 5bc960e..a78c4b7 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:33:15.457 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:49:46.219 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -53,15 +53,16 @@ The are no Supported Operating Systems specified for this project. * [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) * [SubworkflowFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#SubworkflowFeatureRequirement) +* [SchemaDefRequirement](https://www.commonwl.org/v1.2/Workflow.html#SchemaDefRequirement) ### Inputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | -| `stac_items` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Sentinel-2 STAC items | list of Sentinel-2 COG STAC items | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | +| `stac_items` | `array` of [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Sentinel-2 STAC items | list of Sentinel-2 COG STAC items | +| `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | ### Steps @@ -143,15 +144,16 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram ### Requirements * [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) +* [SchemaDefRequirement](https://www.commonwl.org/v1.2/Workflow.html#SchemaDefRequirement) ### Inputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Area of interest | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | Coordinate reference system | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | NDWI bands | bands used for the NDWI | -| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item | STAC item | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| Coordinate reference system | EPSG code | +| `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| NDWI bands | bands used for the NDWI | +| `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item | STAC item | ### Steps @@ -235,10 +237,10 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `item` | `--input-item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `aoi` | `--aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `epsg` | `--epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| +| `band` | `--band` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| ### Execution usage example: @@ -318,7 +320,7 @@ python -m app \ | Id | Option | Type | |----|------|-------| -| `item` | `--item` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `item` | `--item` | `array` of [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| | `rasters` | `--rasters` | `array` of [File](https://www.commonwl.org/v1.2/Workflow.html#File) | ### Execution usage example: diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg index bcae5bd..4f14e55 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/class.svg @@ -1 +1 @@ -Water bodies detection based on NDWI and otsu thresholdStepsWater body detection based on NDWI and otsu thresholdStepsRequirementsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 stac_items: string[] bands: string[] = ['green', 'nir']  Inputs stac_catalog: Directory   Outputs node_water_bodies: detect_water_body node_stac: stac  StepsWorkflowstac item: string[] rasters: File[]  Inputs stac_catalog: Directory   OutputsCommandLineTooldetect_water_body aoi: string epsg: string = EPSG:4326 bands: string[] item: string  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirementScatterFeatureRequirementSubworkflowFeatureRequirement \ No newline at end of file +SchemasWater bodies detection based on NDWI and otsu thresholdStepsWater body detection based on NDWI and otsu thresholdStepsRequirementsRequirementsPointTypePointPointtype: PointType coordinates: double[] bbox: double[]LineStringTypeLineStringLineStringtype: LineStringType coordinates: double[] bbox: double[]PolygonTypePolygonPolygontype: PolygonType coordinates: double[][][] bbox: double[]MultiPointTypeMultiPointMultiPointtype: MultiPointType coordinates: double[][] bbox: double[]MultiLineStringTypeMultiLineStringMultiLineStringtype: MultiLineStringType coordinates: double[][][] bbox: double[]MultiPolygonTypeMultiPolygonMultiPolygontype: MultiPolygonType coordinates: double[][][][] bbox: double[]FeatureTypeFeatureFeaturetype: FeatureType id: string geometry: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon bbox: double[]GeometryCollectionTypeGeometryCollectionGeometryCollectiontype: GeometryCollectionType geometries: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon[] bbox: double[]Linkhref: string rel: string type: string hreflang: string title: string length: intFeatureCollectionTypeFeatureCollectionFeatureCollectiontype: FeatureCollectionType features: Feature[] links: Link[] timeStamp: string numberMatched: int numberReturned: intDatevalue: stringDateTimevalue: stringDurationvalue: stringEmailvalue: stringHostnamevalue: stringIDNEmailvalue: stringIDNHostnamevalue: stringIPv4value: stringIPv6value: stringIRIvalue: stringIRIReferencevalue: stringPasswordvalue: stringJsonPointervalue: stringRelativeJsonPointervalue: stringUUIDvalue: stringURIvalue: stringURIReferencevalue: stringURITemplatevalue: stringTimevalue: stringPatternvalue: string pattern: stringEPSGCode4326SpectralBandgreen nir nir08water-bodies aoi: Polygon epsg: EPSGCode = 4326 stac_items: URI[] bands: SpectralBand[] = ['green', 'nir']  Inputs stac_catalog: Directory   Outputs node_water_bodies: detect_water_body node_stac: stac  StepsWorkflowstac item: URI[] rasters: File[]  Inputs stac_catalog: Directory   OutputsCommandLineTooldetect_water_body aoi: Polygon epsg: EPSGCode = 4326 bands: SpectralBand[] item: URI  Inputs detected_water_body: File   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu  StepsWorkflowcrop item: URI aoi: Polygon epsg: EPSGCode band: SpectralBand  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   OutputsScatterFeatureRequirementSchemaDefRequirementScatterFeatureRequirementSubworkflowFeatureRequirementSchemaDefRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg index 3e8d965..e395bc2 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   stac_items   bands   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSentinel-2 STAC itemsdescriptionlist of Sentinel-2 COG STAC itemstypearrayitems   typestring   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[] \ No newline at end of file +aoi   epsg   stac_items   bands   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSentinel-2 STAC itemsdescriptionlist of Sentinel-2 COG STAC itemstypearrayitems   typestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[] \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md index 15fbf66..960c1b0 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to a single Sent > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:33:17.352 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:49:51.045 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -52,15 +52,16 @@ The are no Supported Operating Systems specified for this project. ### Requirements * [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) +* [SchemaDefRequirement](https://www.commonwl.org/v1.2/Workflow.html#SchemaDefRequirement) ### Inputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | -| `item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | STAC item reference | Reference to a STAC item | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | +| `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | +| `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item reference | Reference to a STAC item | ### Steps @@ -145,10 +146,10 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `item` | `--input-item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `aoi` | `--aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `epsg` | `--epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| +| `band` | `--band` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| ### Execution usage example: @@ -226,7 +227,7 @@ python -m app \ | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `item` | `--input-item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| | `rasters` | `--water-body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | ### Execution usage example: diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg index d473ee0..6b2decb 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/class.svg @@ -1 +1 @@ -Water bodies detection based on NDWI and the otsu thresholdStepsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 bands: string[] = ['green', 'nir'] item: string  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: string aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: string rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirement \ No newline at end of file +SchemasWater bodies detection based on NDWI and the otsu thresholdStepsRequirementsPointTypePointPointtype: PointType coordinates: double[] bbox: double[]LineStringTypeLineStringLineStringtype: LineStringType coordinates: double[] bbox: double[]PolygonTypePolygonPolygontype: PolygonType coordinates: double[][][] bbox: double[]MultiPointTypeMultiPointMultiPointtype: MultiPointType coordinates: double[][] bbox: double[]MultiLineStringTypeMultiLineStringMultiLineStringtype: MultiLineStringType coordinates: double[][][] bbox: double[]MultiPolygonTypeMultiPolygonMultiPolygontype: MultiPolygonType coordinates: double[][][][] bbox: double[]FeatureTypeFeatureFeaturetype: FeatureType id: string geometry: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon bbox: double[]GeometryCollectionTypeGeometryCollectionGeometryCollectiontype: GeometryCollectionType geometries: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon[] bbox: double[]Linkhref: string rel: string type: string hreflang: string title: string length: intFeatureCollectionTypeFeatureCollectionFeatureCollectiontype: FeatureCollectionType features: Feature[] links: Link[] timeStamp: string numberMatched: int numberReturned: intDatevalue: stringDateTimevalue: stringDurationvalue: stringEmailvalue: stringHostnamevalue: stringIDNEmailvalue: stringIDNHostnamevalue: stringIPv4value: stringIPv6value: stringIRIvalue: stringIRIReferencevalue: stringPasswordvalue: stringJsonPointervalue: stringRelativeJsonPointervalue: stringUUIDvalue: stringURIvalue: stringURIReferencevalue: stringURITemplatevalue: stringTimevalue: stringPatternvalue: string pattern: stringEPSGCode4326SpectralBandgreen nir nir08water-bodies aoi: Polygon epsg: EPSGCode = 4326 bands: SpectralBand[] = ['green', 'nir'] item: URI  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: URI aoi: Polygon epsg: EPSGCode band: SpectralBand  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: URI rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirementSchemaDefRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg index 95e58c2..be4fd3d 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemtypestring   titlecwl:typevaluestring \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md index 8b729d7..08f0555 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:33:19.131 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T18:49:54.541 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -52,14 +52,15 @@ The are no Supported Operating Systems specified for this project. ### Requirements * [ScatterFeatureRequirement](https://www.commonwl.org/v1.2/Workflow.html#ScatterFeatureRequirement) +* [SchemaDefRequirement](https://www.commonwl.org/v1.2/Workflow.html#SchemaDefRequirement) ### Inputs | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | area of interest | area of interest as a bounding box | -| `epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | EPSG code | EPSG code | -| `bands` | `array` of [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | bands used for the NDWI | bands used for the NDWI | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | +| `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | | `item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | STAC item reference | Reference to a STAC item | @@ -146,9 +147,9 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Option | Type | |----|------|-------| | `item` | `--input-item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | -| `aoi` | `--aoi` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `epsg` | `--epsg` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | -| `band` | `--band` | [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType) | +| `aoi` | `--aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `epsg` | `--epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| +| `band` | `--band` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| ### Execution usage example: diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg index 17bbc15..e50a71d 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/class.svg @@ -1 +1 @@ -Water body detection based on NDWI and the otsu thresholdStepsRequirementswater-bodies aoi: string epsg: string = EPSG:4326 bands: string[] = ['green', 'nir'] item: Directory  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: Directory aoi: string epsg: string band: string  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: Directory rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirement \ No newline at end of file +SchemasWater body detection based on NDWI and the otsu thresholdStepsRequirementsPointTypePointPointtype: PointType coordinates: double[] bbox: double[]LineStringTypeLineStringLineStringtype: LineStringType coordinates: double[] bbox: double[]PolygonTypePolygonPolygontype: PolygonType coordinates: double[][][] bbox: double[]MultiPointTypeMultiPointMultiPointtype: MultiPointType coordinates: double[][] bbox: double[]MultiLineStringTypeMultiLineStringMultiLineStringtype: MultiLineStringType coordinates: double[][][] bbox: double[]MultiPolygonTypeMultiPolygonMultiPolygontype: MultiPolygonType coordinates: double[][][][] bbox: double[]FeatureTypeFeatureFeaturetype: FeatureType id: string geometry: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon bbox: double[]GeometryCollectionTypeGeometryCollectionGeometryCollectiontype: GeometryCollectionType geometries: Point | LineString | Polygon | MultiPoint | MultiLineString | MultiPolygon[] bbox: double[]Linkhref: string rel: string type: string hreflang: string title: string length: intFeatureCollectionTypeFeatureCollectionFeatureCollectiontype: FeatureCollectionType features: Feature[] links: Link[] timeStamp: string numberMatched: int numberReturned: intEPSGCode4326SpectralBandgreen nir nir08water-bodies aoi: Polygon epsg: EPSGCode = 4326 bands: SpectralBand[] = ['green', 'nir'] item: Directory  Inputs stac_catalog: Directory   Outputs node_crop: crop node_normalized_difference: norm_diff node_otsu: otsu node_stac: stac  StepsWorkflowcrop item: Directory aoi: Polygon epsg: EPSGCode band: SpectralBand  Inputs cropped: File   OutputsCommandLineToolnorm_diff rasters: File[]  Inputs ndwi: File   Outputsotsu raster: File  Inputs binary_mask_item: File   Outputsstac item: Directory rasters: File  Inputs stac_catalog: Directory   OutputsScatterFeatureRequirementSchemaDefRequirement \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg index 1a3d552..f65f233 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionarea of interest as a bounding boxtypestring   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringdefaultEPSG:4326   titlecwl:typevaluestringschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringgreennir   titlecwl:typevaluestring[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemoneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemoneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/index.md b/docs/reference/cwl/index.md index a726cd6..0fd6e7b 100644 --- a/docs/reference/cwl/index.md +++ b/docs/reference/cwl/index.md @@ -75,3 +75,29 @@ its workflow-selection requirement. Container declarations now use mandatory `DockerRequirement` entries rather than advisory hints, so runners must honor the specified images. + +## Typed EOAP inputs + +The CWL documents selectively adopt the semantic typing from +`feature/enhancements`: imported GeoJSON `Polygon` and string-format `URI` +records, an EPSG enum (`4326`), and spectral-band enums (`green`, `nir`, `nir08`). +Staged acquisition inputs retain their `Directory` type. + +Use `cwl-workflow/typed-cloud-native-inputs.yaml` as an input-shape example: + +```sh +cwltool --podman cwl-workflow/app-water-body-cloud-native.cwl \ + cwl-workflow/typed-cloud-native-inputs.yaml +``` + +The example retains the tutorial's STAC reference; its availability is not +verified by static validation. For the multi-acquisition workflow, supply +`stac_items` as a list of records with a `value` field instead of `item`. + +The imported Polygon schema requires both `coordinates` and `bbox`. The crop +binding passes `bbox` as comma-separated coordinates to the existing application; +cropping remains rectangular. URI records are unwrapped through `self.value`. +These changes require updating jobs that previously supplied plain AOI and STAC +strings. The imports reference the schemas repository's `main` branch. + +Workflow enum definitions are shared through `cwl-workflow/eoap-types.yaml` and imported with `SchemaDefRequirement`. Named enum references allow cwltool to bind array inputs and preserve identical types across workflow/tool boundaries. diff --git a/practice-labs/3-CWL-CommandLineTools/crop.ipynb b/practice-labs/3-CWL-CommandLineTools/crop.ipynb index c2dcbe5..8e31a14 100644 --- a/practice-labs/3-CWL-CommandLineTools/crop.ipynb +++ b/practice-labs/3-CWL-CommandLineTools/crop.ipynb @@ -12,6 +12,13 @@ "\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -78,16 +85,37 @@ }, "outputs": [], "source": [ - "\n", + "cat <<'EOF' > crop-green-params.yaml\n", + "item:\n", + " value: \n", + " https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n", + "aoi:\n", + " type: Polygon\n", + " coordinates:\n", + " - - - -121.399\n", + " - 39.834\n", + " - - -120.74\n", + " - 39.834\n", + " - - -120.74\n", + " - 40.472\n", + " - - -121.399\n", + " - 40.472\n", + " - - -121.399\n", + " - 39.834\n", + " bbox:\n", + " - -121.399\n", + " - 39.834\n", + " - -120.74\n", + " - 40.472\n", + "epsg: EPSG:4326\n", + "band: green\n", + "EOF\n", "\n", "cwltool \\\n", " --podman \\\n", " --outdir ${WORKSPACE}/runs \\\n", " ${WORKSPACE}/cwl-cli/crop.cwl \\\n", - " --item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --aoi=\"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\" \\\n", - " --band \"green\" 2> crop.log" + " crop-green-params.yaml 2> crop.log\n" ] }, { @@ -127,12 +155,31 @@ }, "outputs": [], "source": [ - "cat << EOF > params.yaml\n", - "item: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n", - "aoi: \"-121.399,39.834,-120.74,40.472\"\n", - "epsg: \"EPSG:4326\"\n", + "cat <<'EOF' > params.yaml\n", + "item:\n", + " value: \n", + " https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n", + "aoi:\n", + " type: Polygon\n", + " coordinates:\n", + " - - - -121.399\n", + " - 39.834\n", + " - - -120.74\n", + " - 39.834\n", + " - - -120.74\n", + " - 40.472\n", + " - - -121.399\n", + " - 40.472\n", + " - - -121.399\n", + " - 39.834\n", + " bbox:\n", + " - -121.399\n", + " - 39.834\n", + " - -120.74\n", + " - 40.472\n", + "epsg: EPSG:4326\n", "band: nir\n", - "EOF" + "EOF\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb b/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb index ae08a68..9f28257 100644 --- a/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb +++ b/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb @@ -14,15 +14,22 @@ "https://eoap.github.io/mastering-app-package/cwl-workflow/cloud-native/\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "The Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a single STAC item as input parameters:\n", "\n", - "- a SpatioTemporal Asset Catalog (STAC) Item\n", - "- a bounding box area of interest (AOI)\n", - "- the EPSG code of the bounding box area of interest\n", + "- a SpatioTemporal Asset Catalog (STAC) Item URI record (`value: `)\n", + "- a GeoJSON Polygon area of interest (AOI), including `coordinates` and `bbox`\n", + "- the EPSG enum value `\"4326\"` for the area of interest\n", "- a list of common band names ([\"`green`\", \"`nir`\"])" ] }, @@ -122,9 +129,9 @@ "\n", "These are the Application Package inputs: \n", "\n", - "- a SpatioTemporal Asset Catalog (STAC) Item: `item`\n", - "- a bounding box area of interest (AOI): `aoi`\n", - "- the EPSG code of the bounding box area of interest: `epsg`\n", + "- a SpatioTemporal Asset Catalog (STAC) Item URI record (`value: `): `item`\n", + "- a GeoJSON Polygon area of interest (AOI), including `coordinates` and `bbox`: `aoi`\n", + "- the EPSG enum value `\"4326\"` for the area of interest: `epsg`\n", "- a list of common band names ([\"`green`\", \"`nir`\"]): `bands`" ] }, @@ -299,14 +306,11 @@ "outputs": [], "source": [ "cwltool \\\n", + " --podman \\\n", " --parallel \\\n", " --outdir ${WORKSPACE}/runs \\\n", " ${WORKSPACE}/cwl-workflow/app-water-body-cloud-native.cwl#water-bodies \\\n", - " --item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --aoi=\"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\" \\\n", - " --bands green \\\n", - " --bands nir" + " ${WORKSPACE}/cwl-workflow/typed-cloud-native-inputs.yaml\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/2.2-staged.ipynb b/practice-labs/4-CWL-Workflows/2.2-staged.ipynb index 03e78ab..dde8287 100644 --- a/practice-labs/4-CWL-Workflows/2.2-staged.ipynb +++ b/practice-labs/4-CWL-Workflows/2.2-staged.ipynb @@ -11,11 +11,18 @@ "The non-Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a staged EO acquisition as a STAC Catalog in a directory as input parameters:\n", "\n", "- a local STAC Catalog in a directory\n", - "- a bounding box area of interest (AOI)\n", - "- the EPSG code of the bounding box area of interest\n", + "- a GeoJSON Polygon area of interest (AOI), including `coordinates` and `bbox`\n", + "- the EPSG enum value `\"4326\"` for the area of interest\n", "- a list of common band names ([\"`green`\", \"`nir08`\"])" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -109,15 +116,39 @@ }, "outputs": [], "source": [ + "cat < staged-params.yaml\n", + "item:\n", + " class: Directory\n", + " path: \"$(jq -r .staged.path staged.json)\"\n", + "aoi:\n", + " type: Polygon\n", + " coordinates:\n", + " - - - -118.985\n", + " - 38.432\n", + " - - -118.183\n", + " - 38.432\n", + " - - -118.183\n", + " - 38.938\n", + " - - -118.985\n", + " - 38.938\n", + " - - -118.985\n", + " - 38.432\n", + " bbox:\n", + " - -118.985\n", + " - 38.432\n", + " - -118.183\n", + " - 38.938\n", + "epsg: '4326'\n", + "bands:\n", + " - green\n", + " - nir08\n", + "EOF\n", + "\n", "cwltool \\\n", " --podman \\\n", " --outdir ${WORKSPACE}/runs \\\n", " ${WORKSPACE}/cwl-workflow/app-water-body.cwl#water-bodies \\\n", - " --item $( cat staged.json | jq -r .staged.path ) \\\n", - " --aoi=\"-118.985,38.432,-118.183,38.938\" \\\n", - " --epsg \"EPSG:4326\" \\\n", - " --bands green \\\n", - " --bands nir08" + " staged-params.yaml\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb b/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb index 88acc97..a81eaa3 100644 --- a/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb +++ b/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb @@ -14,15 +14,22 @@ "https://eoap.github.io/mastering-app-package/cwl-workflow/scatter-cloud-native/" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "The Cloud native Workflow chains the `crop`, `norm_diff`, `otsu` and `stac` steps taking a single STAC item as input parameters:\n", "\n", - "- a SpatioTemporal Asset Catalog (STAC) Item\n", - "- a bounding box area of interest (AOI)\n", - "- the EPSG code of the bounding box area of interest\n", + "- a SpatioTemporal Asset Catalog (STAC) Item URI record (`value: `)\n", + "- a GeoJSON Polygon area of interest (AOI), including `coordinates` and `bbox`\n", + "- the EPSG enum value `\"4326\"` for the area of interest\n", "- a list of common band names ([\"`green`\", \"`nir`\"])\n", "\n", "CWL can run sub-workflows in a step.\n", @@ -150,9 +157,9 @@ "\n", "These are the Application Package inputs: \n", "\n", - "- a list of SpatioTemporal Asset Catalog (STAC) Items: `stac_items` of type `string[]`\n", - "- a bounding box area of interest (AOI): `aoi`\n", - "- the EPSG code of the bounding box area of interest: `epsg`\n", + "- a list of SpatioTemporal Asset Catalog (STAC) Items: `stac_items` as an array of URI records, each with a `value` field\n", + "- a GeoJSON Polygon area of interest (AOI), including `coordinates` and `bbox`: `aoi`\n", + "- the EPSG enum value `\"4326\"` for the area of interest: `epsg`\n", "- a list of common band names ([\"`green`\", \"`nir`\"]): `bands`" ] }, @@ -253,14 +260,10 @@ "outputs": [], "source": [ "cwltool \\\n", + " --podman \\\n", " --outdir ${WORKSPACE}/runs \\\n", " ${WORKSPACE}/cwl-workflow/app-water-bodies-cloud-native.cwl#water-bodies \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210723_0_L2A\" \\\n", - " --aoi=\"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\" \\\n", - " --bands green \\\n", - " --bands nir" + " ${WORKSPACE}/cwl-workflow/typed-scatter-inputs.yaml\n" ] }, { diff --git a/scripts/cwl-cli-crop-green.sh b/scripts/cwl-cli-crop-green.sh index ca592f1..ef4c55c 100644 --- a/scripts/cwl-cli-crop-green.sh +++ b/scripts/cwl-cli-crop-green.sh @@ -1,13 +1,8 @@ -export WORKSPACE=/workspace/mastering-app-package -export RUNTIME=${WORKSPACE}/runs -mkdir -p ${RUNTIME} -cd ${RUNTIME} +export WORKSPACE=${WORKSPACE:-/workspace/mastering-app-package} +mkdir -p "${WORKSPACE}/runs" cwltool \ --podman \ - --outdir ${WORKSPACE}/runs \ - ${WORKSPACE}/cwl-cli/crop.cwl \ - --item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --aoi="-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ - --band "green" \ No newline at end of file + --outdir "${WORKSPACE}/runs" \ + "${WORKSPACE}/cwl-cli/crop.cwl" \ + "${WORKSPACE}/cwl-cli/crop-green-params.yaml" diff --git a/scripts/cwl-cli-crop-nir.sh b/scripts/cwl-cli-crop-nir.sh index 6552d6b..daaefd9 100644 --- a/scripts/cwl-cli-crop-nir.sh +++ b/scripts/cwl-cli-crop-nir.sh @@ -1,13 +1,8 @@ -export WORKSPACE=/workspace/mastering-app-package -export RUNTIME=${WORKSPACE}/runs -mkdir -p ${RUNTIME} -cd ${RUNTIME} +export WORKSPACE=${WORKSPACE:-/workspace/mastering-app-package} +mkdir -p "${WORKSPACE}/runs" cwltool \ --podman \ - --outdir ${WORKSPACE}/runs \ - ${WORKSPACE}/cwl-cli/crop.cwl \ - --item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --aoi="-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ - --band "nir" \ No newline at end of file + --outdir "${WORKSPACE}/runs" \ + "${WORKSPACE}/cwl-cli/crop.cwl" \ + "${WORKSPACE}/cwl-cli/crop-params.yaml" diff --git a/scripts/cwl-workflow-cloud-native-scatter.sh b/scripts/cwl-workflow-cloud-native-scatter.sh index 2fbad2b..693f662 100644 --- a/scripts/cwl-workflow-cloud-native-scatter.sh +++ b/scripts/cwl-workflow-cloud-native-scatter.sh @@ -1,10 +1,8 @@ -export WORKSPACE=/workspace/mastering-app-package +export WORKSPACE=${WORKSPACE:-/workspace/mastering-app-package} +mkdir -p "${WORKSPACE}/runs" cwltool \ --podman \ - --outdir${WORKSPACE}/runs \ - ${WORKSPACE}/cwl-workflow/app-water-bodies-cloud-native.cwl \ - --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A" \ - --aoi="-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ No newline at end of file + --outdir "${WORKSPACE}/runs" \ + "${WORKSPACE}/cwl-workflow/app-water-bodies-cloud-native.cwl#water-bodies" \ + "${WORKSPACE}/cwl-workflow/typed-scatter-inputs.yaml" diff --git a/scripts/cwl-workflow-staged.sh b/scripts/cwl-workflow-staged.sh index 1fd46bd..4b20bbb 100644 --- a/scripts/cwl-workflow-staged.sh +++ b/scripts/cwl-workflow-staged.sh @@ -1,9 +1,33 @@ +cat < staged-params.yaml +item: + class: Directory + path: "$(jq -r .staged.path staged.json)" +aoi: + type: Polygon + coordinates: + - - - -118.985 + - 38.432 + - - -118.183 + - 38.432 + - - -118.183 + - 38.938 + - - -118.985 + - 38.938 + - - -118.985 + - 38.432 + bbox: + - -118.985 + - 38.432 + - -118.183 + - 38.938 +epsg: '4326' +bands: + - green + - nir08 +EOF + cwltool \ --podman \ - --outdir ${WORKSPACE}/runs \ - ${WORKSPACE}/cwl-workflow/cwl-workflow/app-water-body.cwl \ - --item $( cat staged.json | jq -r .staged.path ) \ - --aoi="-118.985,38.432,-118.183,38.938" \ - --epsg "EPSG:4326" \ - --bands green \ - --bands nir08 \ No newline at end of file + --outdir "${WORKSPACE}/runs" \ + "${WORKSPACE}/cwl-workflow/app-water-body.cwl#water-bodies" \ + staged-params.yaml From 361275e42e86906de4b081875db273461cb5c9da Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 19:51:48 +0200 Subject: [PATCH 07/23] fix(labs): bound remote raster reads and allocate normalization memory --- cwl-cli/norm-diff.cwl | 3 +- .../app-water-bodies-cloud-native.cwl | 3 +- cwl-workflow/app-water-body-cloud-native.cwl | 3 +- cwl-workflow/app-water-body.cwl | 3 +- .../command-line-tools/crop/Dockerfile | 4 +- water-bodies/command-line-tools/crop/app.py | 84 +++++++++++-------- 6 files changed, 61 insertions(+), 39 deletions(-) diff --git a/cwl-cli/norm-diff.cwl b/cwl-cli/norm-diff.cwl index 5fb2465..ccc4c7c 100644 --- a/cwl-cli/norm-diff.cwl +++ b/cwl-cli/norm-diff.cwl @@ -41,7 +41,8 @@ requirements: PYTHONPATH: /app ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 2048 + ramMax: 2048 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index 12aad03..92136ce 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -236,7 +236,8 @@ $graph: PYTHONPATH: /app ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 2048 + ramMax: 2048 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index f790eaf..0d36704 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -181,7 +181,8 @@ $graph: PYTHONPATH: /app ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 2048 + ramMax: 2048 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index f45b3bd..bc06873 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -178,7 +178,8 @@ $graph: PYTHONPATH: /app ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 2048 + ramMax: 2048 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/water-bodies/command-line-tools/crop/Dockerfile b/water-bodies/command-line-tools/crop/Dockerfile index 4039e9d..a8c128c 100644 --- a/water-bodies/command-line-tools/crop/Dockerfile +++ b/water-bodies/command-line-tools/crop/Dockerfile @@ -1,11 +1,11 @@ -FROM docker.io/library/python:3.11-slim@sha256:91e9d01cf4bd56be7128c603506b6fe367ef7506f9f2ad8f3a908aeec8941bb9 +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 # System deps needed by rasterio wheels RUN apt-get update && \ apt-get install -y --no-install-recommends libexpat1 && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir "rasterio==1.4.3" "click==8.2.1" "pystac==1.13.0" "loguru==0.7.3" "pyproj==3.7.2" "shapely==2.1.1" && \ +RUN pip install --no-cache-dir "rasterio==1.5.2" "click==8.5.0" "pystac==1.15.2" "loguru==0.7.3" "pyproj==3.8.0" "shapely==2.1.2" && \ python -c "import rasterio" ADD app.py /app/app.py diff --git a/water-bodies/command-line-tools/crop/app.py b/water-bodies/command-line-tools/crop/app.py index 606fb9b..98d8bb0 100644 --- a/water-bodies/command-line-tools/crop/app.py +++ b/water-bodies/command-line-tools/crop/app.py @@ -1,4 +1,5 @@ import os +import time import click import pystac import rasterio @@ -78,39 +79,56 @@ def crop(item_url, aoi, band, epsg): bbox = aoi2box(aoi) - with rasterio.open(asset.get_absolute_href()) as src: - - transformer = Transformer.from_crs(epsg, src.crs, always_xy=True) - - minx, miny = transformer.transform(bbox[0], bbox[1]) - maxx, maxy = transformer.transform(bbox[2], bbox[3]) - - transformed_bbox = box(minx, miny, maxx, maxy) - - logger.info(f"Crop {asset.get_absolute_href()}") - - out_image, out_transform = rasterio.mask.mask( - src, [transformed_bbox], crop=True - ) - out_meta = src.meta.copy() - - out_meta.update( - { - "height": out_image.shape[1], - "width": out_image.shape[2], - "transform": out_transform, - "dtype": "uint16", - "driver": "COG", - "tiled": True, - "compress": "lzw", - "blockxsize": 256, - "blockysize": 256, - } - ) - - with rasterio.open(f"crop_{band}.tif", "w", **out_meta) as dst_dataset: - logger.info(f"Write crop_{band}.tif") - dst_dataset.write(out_image) + for attempt in range(3): + try: + # Bound remote COG requests so a stalled connection cannot hang a lab. + with rasterio.Env( + GDAL_HTTP_CONNECTTIMEOUT=10, + GDAL_HTTP_TIMEOUT=60, + GDAL_HTTP_MAX_RETRY=2, + GDAL_HTTP_RETRY_DELAY=1, + GDAL_HTTP_RETRY_CODES="ALL", + GDAL_HTTP_MULTIRANGE="SERIAL", + CPL_VSIL_CURL_NON_CACHED=f"/vsicurl/{asset.get_absolute_href()}", + ), rasterio.open(asset.get_absolute_href()) as src: + + transformer = Transformer.from_crs(epsg, src.crs, always_xy=True) + + minx, miny = transformer.transform(bbox[0], bbox[1]) + maxx, maxy = transformer.transform(bbox[2], bbox[3]) + + transformed_bbox = box(minx, miny, maxx, maxy) + + logger.info(f"Crop {asset.get_absolute_href()}") + + out_image, out_transform = rasterio.mask.mask( + src, [transformed_bbox], crop=True + ) + out_meta = src.meta.copy() + + out_meta.update( + { + "height": out_image.shape[1], + "width": out_image.shape[2], + "transform": out_transform, + "dtype": "uint16", + "driver": "COG", + "tiled": True, + "compress": "lzw", + "blockxsize": 256, + "blockysize": 256, + } + ) + + with rasterio.open(f"crop_{band}.tif", "w", **out_meta) as dst_dataset: + logger.info(f"Write crop_{band}.tif") + dst_dataset.write(out_image) + break + except rasterio.errors.RasterioIOError: + if attempt == 2 or not asset.get_absolute_href().startswith(("http://", "https://")): + raise + logger.warning("Remote raster read failed; retrying crop ({}/3)", attempt + 2) + time.sleep(1) logger.info("Done!") From c47e3ac8e7a6bd41f2140e9c4bdeb13f4f878337 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:31:25 +0200 Subject: [PATCH 08/23] fix(labs): validate checked-out typed CWL with Calrissian --- practice-labs/5-Kubernetes/1-calrissian.ipynb | 146 +++++--------- practice-labs/5-Kubernetes/2-benchmark.ipynb | 189 +++++------------- practice-labs/5-Kubernetes/README.md | 43 ++++ practice-labs/5-Kubernetes/k8s-job.yaml | 15 +- 4 files changed, 143 insertions(+), 250 deletions(-) create mode 100644 practice-labs/5-Kubernetes/README.md diff --git a/practice-labs/5-Kubernetes/1-calrissian.ipynb b/practice-labs/5-Kubernetes/1-calrissian.ipynb index a92256d..8f3fa71 100644 --- a/practice-labs/5-Kubernetes/1-calrissian.ipynb +++ b/practice-labs/5-Kubernetes/1-calrissian.ipynb @@ -8,7 +8,7 @@ "\n", "## Goal\n", "\n", - "Run the `app-water-body-cloud-native.1.0.0.cwl` released application package using `calrissian`, a CWL runner for kubernetes.\n", + "Run the application package from the current checkout using `calrissian`, a CWL runner for kubernetes.\n", "\n", "While `cwltool` creates a container for each command line tool, `calrissian` creates a pod for each command line tool of the Workflow processing step.\n", "\n", @@ -34,7 +34,6 @@ "source": [ "export WORKSPACE=/workspace/mastering-app-package\n", "export RUNTIME=${WORKSPACE}/runs\n", - "rm -fr ${WORKSPACE}/runs\n", "mkdir -p ${RUNTIME}\n", "cd ${RUNTIME}" ] @@ -43,7 +42,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Download the released Application package" + "## Pack the checked-out application package\n", + "\n", + "Build and load the local tool images first; see the README in this folder.\n", + "Packing names the selected workflow entrypoint `main`.\n" ] }, { @@ -56,11 +58,28 @@ }, "outputs": [], "source": [ - "version=\"1.1.0\"\n", - "\n", - "wget \\\n", - " -O ${WORKSPACE}/runs/app-water-bodies-cloud-native.${version}.cwl \\\n", - " https://github.com/eoap/mastering-app-package/releases/download/${version}/app-water-bodies-cloud-native.${version}.cwl" + "mkdir -p /calrissian\n", + "cwltool --pack \"${WORKSPACE}/cwl-workflow/app-water-bodies-cloud-native.cwl#water-bodies\" > /calrissian/app-water-bodies-cloud-native.cwl\n", + "# A non-latest tag lets Kubernetes use images loaded into the local node.\n", + "python - <<'PYCODE'\n", + "import json\n", + "from pathlib import Path\n", + "path = Path(\"/calrissian/app-water-bodies-cloud-native.cwl\")\n", + "package = json.loads(path.read_text())\n", + "def use_local_images(value):\n", + " if isinstance(value, dict):\n", + " for key, child in value.items():\n", + " if key == \"dockerPull\" and isinstance(child, str) and child.startswith(\"localhost/\") and child.endswith(\":latest\"):\n", + " value[key] = child.removesuffix(\":latest\") + \":metadata\"\n", + " else:\n", + " use_local_images(child)\n", + " elif isinstance(value, list):\n", + " for child in value:\n", + " use_local_images(child)\n", + "use_local_images(package)\n", + "path.write_text(json.dumps(package, indent=2))\n", + "PYCODE\n", + "cp \"${WORKSPACE}/cwl-workflow/typed-scatter-inputs.yaml\" /calrissian/params.yaml\n" ] }, { @@ -97,28 +116,18 @@ "outputs": [], "source": [ "mkdir -p /calrissian/logs\n", - "\n", - "version=\"1.1.0\"\n", - "\n", "calrissian \\\n", " --stdout /calrissian/results.json \\\n", " --stderr /calrissian/app.log \\\n", - " --max-ram 4G \\\n", - " --max-cores \"8\" \\\n", + " --max-ram 3G \\\n", + " --max-cores 2 \\\n", " --tmp-outdir-prefix /calrissian/tmp \\\n", " --outdir /calrissian/results \\\n", " --usage-report /calrissian/usage.json \\\n", " --tool-logs-basepath /calrissian/logs \\\n", " --pod-nodeselectors /etc/calrissian/pod-node-selector.yaml \\\n", - " /workspace/mastering-app-package/runs/app-water-bodies-cloud-native.${version}.cwl#water-bodies \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210708_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210718_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220514_0_L2A\" \\\n", - " --stac_items \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220504_0_L2A\" \\\n", - " --aoi=\"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\"" + " /calrissian/app-water-bodies-cloud-native.cwl#main \\\n", + " /calrissian/params.yaml\n" ] }, { @@ -184,37 +193,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Prepare the parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "cat << EOF > /calrissian/params.yaml\n", - "stac_items:\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210708_0_L2A\"\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\"\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210718_0_L2A\"\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A\"\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220514_0_L2A\"\n", - "- \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220504_0_L2A\"\n", - "aoi: -121.399,39.834,-120.74,40.472\n", - "epsg: \"EPSG:4326\"\n", - "EOF" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Put all needed files in the RWX volume:" + "Submit the Job and wait for success or failure:\n" ] }, { @@ -228,53 +207,22 @@ "outputs": [], "source": [ "cp /etc/calrissian/pod-node-selector.yaml /calrissian/pod-node-selector.yaml\n", - "cp ${WORKSPACE}/runs/app-water-bodies-cloud-native.${version}.cwl /calrissian/app-water-bodies-cloud-native.${version}.cwl " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Submit the job:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "kubectl apply -f ${WORKSPACE}/practice-labs/5-Kubernetes/k8s-job.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Monitor the job until completion" + "# Recreate the Job so repeated notebook execution runs it again.\n", + "kubectl delete job water-bodies-detection --ignore-not-found\n", + "kubectl apply -f \"${WORKSPACE}/practice-labs/5-Kubernetes/k8s-job.yaml\"\n", + "completed=false\n", + "for attempt in {1..240}; do\n", + " if kubectl wait --for=condition=complete --timeout=5s job/water-bodies-detection 2>/dev/null; then\n", + " completed=true\n", + " break\n", + " fi\n", + " if [ \"$(kubectl get job water-bodies-detection -o jsonpath='{.status.failed}')\" = \"1\" ]; then\n", + " kubectl logs job/water-bodies-detection --tail=100\n", + " break\n", + " fi\n", + "done\n", + "[ \"$completed\" = true ]\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [], - "source": [ - "kubectl wait --for=condition=complete --timeout=600s job/water-bodies-detection" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] } ], "metadata": { diff --git a/practice-labs/5-Kubernetes/2-benchmark.ipynb b/practice-labs/5-Kubernetes/2-benchmark.ipynb index a89b8d3..e26ddef 100644 --- a/practice-labs/5-Kubernetes/2-benchmark.ipynb +++ b/practice-labs/5-Kubernetes/2-benchmark.ipynb @@ -6,13 +6,13 @@ "source": [ "## Goal\n", "\n", - "Run the app-water-body-cloud-native.1.0.0.cwl released application package using calrissian, a CWL runner for kubernetes and exploit the usage report.\n", + "Run the checked-out application package prepared by `1-calrissian.ipynb` using calrissian, a CWL runner for kubernetes and exploit the usage report.\n", "\n", "## Lab\n", "\n", - "This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Kubernetes/benchmark.ipynb\n", + "This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/5-Kubernetes/2-benchmark.ipynb\n", "\n", - "Select the Python kernel `.venv (Python 3.9.18)` from the top-right.\n" + "Select the Python kernel `Python (Mastering Application Package)` from the top-right.\n" ] }, { @@ -27,37 +27,45 @@ "source": [ "import random\n", "import json\n", + "import subprocess\n", + "import tempfile\n", + "from pathlib import Path\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.dates as mdates\n", - "import json\n", - "import logging\n", - "import unittest\n", - "from io import StringIO\n", - "from pathlib import Path\n", - "from calrissian.context import CalrissianLoadingContext, CalrissianRuntimeContext\n", - "from calrissian.executor import ThreadPoolJobExecutor\n", - "from calrissian.k8s import delete_pods\n", - "from calrissian.main import (\n", - " activate_logging,\n", - " add_arguments,\n", - " flush_tees,\n", - " install_signal_handler,\n", - ")\n", - "from calrissian.report import (\n", - " CPUParser,\n", - " MemoryParser,\n", - " Reporter,\n", - " initialize_reporter,\n", - " default_serializer,\n", - ")\n", - "from calrissian.version import version\n", - "from cwltool.argparser import arg_parser\n", - "from cwltool.main import main as cwlmain\n", - "from nose2.tools import params\n", - "import uuid\n", - "from cwltool.load_tool import fetch_document" + "import yaml\n", + "\n", + "class Benchmark:\n", + " def __init__(self, parameters, max_cores=\"2\", max_ram=\"3G\"):\n", + " self.parameters = parameters\n", + " self.max_cores = max_cores\n", + " self.max_ram = max_ram\n", + " self.cwl_path = Path(\"/calrissian/app-water-bodies-cloud-native.cwl\")\n", + "\n", + " def run(self):\n", + " # Use the public CLI and its usage report instead of internal Python APIs.\n", + " directory = Path(tempfile.mkdtemp(prefix=\"benchmark-\", dir=\"/calrissian\"))\n", + " (directory / \"logs\").mkdir()\n", + " parameters = directory / \"params.yaml\"\n", + " parameters.write_text(yaml.safe_dump(self.parameters))\n", + " subprocess.run([\n", + " \"calrissian\", \"--max-cores\", self.max_cores, \"--max-ram\", self.max_ram,\n", + " \"--tmp-outdir-prefix\", str(directory / \"tmp\"),\n", + " \"--outdir\", str(directory / \"out\"),\n", + " \"--stdout\", str(directory / \"results.json\"),\n", + " \"--stderr\", str(directory / \"app.log\"),\n", + " \"--usage-report\", str(directory / \"usage.json\"),\n", + " \"--tool-logs-basepath\", str(directory / \"logs\"),\n", + " \"--pod-nodeselectors\", \"/etc/calrissian/pod-node-selector.yaml\",\n", + " f\"{self.cwl_path}#main\", str(parameters),\n", + " ], check=True)\n", + " return {\n", + " \"application_package\": {\"uri\": str(self.cwl_path), \"cwl\": json.loads(self.cwl_path.read_text())},\n", + " \"results\": json.loads((directory / \"results.json\").read_text()),\n", + " \"usage\": json.loads((directory / \"usage.json\").read_text()),\n", + " \"parameters\": self.parameters,\n", + " }\n" ] }, { @@ -81,106 +89,9 @@ }, "outputs": [], "source": [ - "class Benchmark():\n", - "\n", - " def __init__(self, aoi, epsg, stac_items, bands, max_cores, max_ram):\n", - "\n", - " self.level = logging.INFO\n", - " \n", - " self.cwl_url = \"https://github.com/eoap/mastering-app-package/releases/download/1.1.0/app-water-bodies-cloud-native.1.1.0.cwl\"\n", - "\n", - " self.tmp_outdir_prefix = \"/calrissian/tmp\"\n", - " self.outdir = \"/calrissian/out\"\n", - "\n", - " self.parser = arg_parser()\n", - "\n", - " self.aoi = aoi \n", - " self.epsg = epsg \n", - " self.stac_items = stac_items\n", - " self.bands = bands \n", - " self.max_cores = max_cores\n", - " self.max_ram = max_ram\n", - "\n", - " def get_cli_args(self):\n", - "\n", - " cli_args = [\n", - " \"--tmp-outdir-prefix\",\n", - " self.tmp_outdir_prefix,\n", - " \"--outdir\",\n", - " self.outdir,\n", - " f\"{self.cwl_url}#water-bodies\",\n", - " f\"--aoi={self.aoi}\",\n", - " f\"--epsg={self.epsg}\"\n", - " ]\n", - "\n", - " for stac_item in self.stac_items:\n", - " cli_args.append(f\"--stac_items={stac_item}\")\n", - "\n", - " for band in self.bands:\n", - " cli_args.append(f\"--band={band}\")\n", - "\n", - " return cli_args\n", - "\n", - " def run(self):\n", - "\n", - " parsed_args = self.parser.parse_args(\n", - " self.get_cli_args()\n", - " )\n", - "\n", - " results, usage = self.run_calrissian(\n", - " parsed_args=parsed_args\n", - " )\n", - "\n", - " _, workflowobj, uri = fetch_document(self.cwl_url)\n", - "\n", - " result = {}\n", - "\n", - " result[\"application_package\"] = {\"uri\": uri, \"cwl\": workflowobj}\n", - " result[\"results\"] = results\n", - " result[\"usage\"] = usage\n", - " result[\"parameters\"] = {\n", - " \"stac_items\": self.stac_items,\n", - " \"aoi\": self.aoi,\n", - " \"epsg\": self.epsg,\n", - " \"bands\": self.bands,\n", - " }\n", - "\n", - " return result\n", - "\n", - "\n", - " def run_calrissian(self, parsed_args):\n", - "\n", - " max_ram_megabytes = MemoryParser.parse_to_megabytes(self.max_ram)\n", - " max_cores = CPUParser.parse(self.max_cores)\n", - " executor = ThreadPoolJobExecutor(max_ram_megabytes, max_cores)\n", - " initialize_reporter(max_ram_megabytes, max_cores)\n", - "\n", - " runtime_context = CalrissianRuntimeContext(vars(parsed_args))\n", - " runtime_context.select_resources = executor.select_resources\n", - "\n", - " install_signal_handler()\n", - "\n", - " stream_out = StringIO()\n", - "\n", - " try:\n", - "\n", - " res = cwlmain(\n", - " args=parsed_args,\n", - " stdout=stream_out,\n", - " executor=executor,\n", - " loadingContext=CalrissianLoadingContext(),\n", - " runtimeContext=runtime_context,\n", - " versionfunc=version,\n", - " logger_handler=logging.FileHandler(\"empty.log\"),\n", - " )\n", - " finally:\n", - " delete_pods()\n", - "\n", - " assert res == 0\n", - "\n", - " return json.loads(stream_out.getvalue()), Reporter.get_report().to_dict()\n", - "\n", - " \n" + "parameters = yaml.safe_load(Path(\"/workspace/mastering-app-package/cwl-workflow/typed-scatter-inputs.yaml\").read_text())\n", + "benchmark = Benchmark(parameters)\n", + "report = benchmark.run()\n" ] }, { @@ -200,19 +111,9 @@ }, "outputs": [], "source": [ - "benchmark = Benchmark(stac_items=[\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210708_0_L2A\",\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\",\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210718_0_L2A\",\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A\",\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220514_0_L2A\",\n", - " \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220504_0_L2A\",\n", - " ],\n", - " aoi=\"-121.399,39.834,-120.74,40.472\",\n", - " epsg=\"EPSG:4326\",\n", - " bands=[\"green\", \"nir\"],\n", - " max_cores=\"20\",\n", - " max_ram=\"128G\")\n" + "# Repeat with reversed scatter inputs to compare another run.\n", + "parameters = dict(parameters, stac_items=list(reversed(parameters[\"stac_items\"])))\n", + "benchmark = Benchmark(parameters)\n" ] }, { @@ -257,7 +158,7 @@ " for elem in cwl[\"$graph\"]:\n", "\n", " if elem[\"class\"] in [\"CommandLineTool\"]:\n", - " color_mapping[elem[\"id\"]] = (\n", + " color_mapping[elem[\"id\"].removeprefix(\"#\")] = (\n", " random.uniform(0, 1),\n", " random.uniform(0, 1),\n", " random.uniform(0, 1),\n", diff --git a/practice-labs/5-Kubernetes/README.md b/practice-labs/5-Kubernetes/README.md new file mode 100644 index 0000000..747a57b --- /dev/null +++ b/practice-labs/5-Kubernetes/README.md @@ -0,0 +1,43 @@ +# Test the checked-out CWL on Minikube + +These notebooks use the current checkout, including `feature/metadata`, rather +than downloading an older released application package. No new CWL release is +needed for local validation. Notebook 1 packs the typed workflow and rewrites +its container tags to `localhost/:metadata`; notebook 2 benchmarks that +same package through the Calrissian CLI and its JSON usage report. + +With the editor deployed by `dev-platform-eoap/mastering-app-package`, run from +that deployment module directory: + +```bash +task labs LAB='[1-4]*' MINIKUBE_PROFILE=eoap-mastering-app-package-docker +task labs:kubernetes-images MINIKUBE_PROFILE=eoap-mastering-app-package-docker +task labs LAB=5-Kubernetes MINIKUBE_PROFILE=eoap-mastering-app-package-docker +``` + +Chapter 2 builds the four tool images using Podman inside the editor. The image +loading task copies those images into the Minikube node; the `metadata` tag +avoids Kubernetes' default pull behavior for `latest`. Run the loading task +again after rebuilding an image. Notebook 1 runs both the installed Calrissian +command and a Kubernetes Job using the same local PDE image as the editor. +The local Job example uses `eoepca/pde-code-server:amd64`; change its image when +using a different PDE build. Each notebook uses the checked-out typed scatter +parameters. The Job is recreated on repeat runs, and the benchmark stores each +run's parameters, outputs, tool logs, and usage report under `/calrissian/benchmark-*`. +The Job retains its status for an hour, and the notebook reports a failed Job +instead of waiting only for success. Retry the lab explicitly after a transient +network failure. + +Published deployments still need accessible, versioned container images and a +published or otherwise distributed application package. This local test path +does not publish images, a CWL release, or the working branch. + +Validation on 10 October 2026 passed all 18 practice notebooks on the Docker +Minikube profile, including the direct Calrissian run, Kubernetes Job, benchmark +workflows, Gantt chart, and report tables. The tested tool snapshot was +`3292d39` plus the fixes in `361275e`, with the notebook changes documented here; +concurrent tool-packaging edits were not copied into that deployment. Remote +raster reads needed bounded retries, and an earlier Job attempt failed during +a transient external DNS outage. Final successful notebook outputs are in +`/workspace/lab-results/20261010T180151267666Z/` (Calrissian) and +`/workspace/lab-results/20261010T180519884005Z/` (benchmark). diff --git a/practice-labs/5-Kubernetes/k8s-job.yaml b/practice-labs/5-Kubernetes/k8s-job.yaml index 1380be7..bb5d97d 100644 --- a/practice-labs/5-Kubernetes/k8s-job.yaml +++ b/practice-labs/5-Kubernetes/k8s-job.yaml @@ -4,7 +4,7 @@ kind: Job metadata: name: water-bodies-detection spec: - ttlSecondsAfterFinished: 5 + ttlSecondsAfterFinished: 3600 template: spec: serviceAccountName: calrissian-sa @@ -14,8 +14,9 @@ spec: fsGroup: 0 containers: - name: calrissian - image: ghcr.io/duke-gcb/calrissian/calrissian:0.18.1 - command: ["calrissian"] + image: eoepca/pde-code-server:amd64 + imagePullPolicy: IfNotPresent + command: ["/opt/calrissian-venv/bin/calrissian"] args: - --debug - --stdout @@ -23,9 +24,9 @@ spec: - --stderr - /calrissian/app.log - --max-ram - - 4G + - 3G - --max-cores - - "8" + - "2" - --tmp-outdir-prefix - /calrissian/tmp/ - --outdir @@ -36,7 +37,7 @@ spec: - /calrissian/logs - --pod-nodeselectors - /calrissian/pod-node-selector.yaml - - /calrissian/app-water-bodies-cloud-native.1.1.0.cwl#water-bodies + - /calrissian/app-water-bodies-cloud-native.cwl#main - /calrissian/params.yaml volumeMounts: - name: calrissian-volume @@ -51,4 +52,4 @@ spec: - name: calrissian-volume persistentVolumeClaim: claimName: calrissian-claim - backoffLimit: 3 + backoffLimit: 0 From 9a3ed61923b228a33d949f434b76be011319397c Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:34:31 +0200 Subject: [PATCH 09/23] feat(processing)!: adopt packaged Python tools and console commands Build processing wheels in multistage containers, preserve dependency pins and remote-read retries, and align CWL bindings with polygon masking and explicit raster options. BREAKING CHANGE: processing tools require Python 3.12 or newer and installed console commands. Rebuild local images; normalized difference and STAC use repeated raster options, and Otsu uses --raster. --- cwl-cli/crop.cwl | 10 +- cwl-cli/norm-diff.cwl | 11 +- cwl-cli/otsu.cwl | 6 +- cwl-cli/stac.cwl | 9 +- .../app-water-bodies-cloud-native.cwl | 39 ++-- cwl-workflow/app-water-body-cloud-native.cwl | 37 ++-- cwl-workflow/app-water-body.cwl | 37 ++-- docs/crop.cwl | 8 +- tests/test_processing_pipeline.py | 46 +++++ .../command-line-tools/crop/Dockerfile | 30 ++- .../command-line-tools/crop/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/crop/README.md | 80 ++++++++ water-bodies/command-line-tools/crop/app.py | 136 +------------- .../command-line-tools/crop/pyproject.toml | 146 +++++++++++++++ .../crop/src/crop/__about__.py | 4 + .../crop/src/crop/__init__.py | 3 + .../command-line-tools/crop/src/crop/cli.py | 72 ++++++++ .../crop/src/crop/crop_impl.py | 157 ++++++++++++++++ .../command-line-tools/crop/tests/__init__.py | 3 + .../crop/tests/test_crop_impl.py | 148 +++++++++++++++ .../command-line-tools/norm_diff/Dockerfile | 29 ++- .../command-line-tools/norm_diff/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/norm_diff/README.md | 50 +++++ .../command-line-tools/norm_diff/app.py | 47 +---- .../norm_diff/pyproject.toml | 144 +++++++++++++++ .../norm_diff/src/norm_diff/__about__.py | 1 + .../norm_diff/src/norm_diff/__init__.py | 1 + .../norm_diff/src/norm_diff/cli.py | 45 +++++ .../norm_diff/src/norm_diff/norm_diff_impl.py | 43 +++++ .../norm_diff/tests/__init__.py | 0 .../norm_diff/tests/test_norm_diff.py | 68 +++++++ .../command-line-tools/otsu/Dockerfile | 28 ++- .../command-line-tools/otsu/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/otsu/README.md | 50 +++++ water-bodies/command-line-tools/otsu/app.py | 46 +---- .../command-line-tools/otsu/pyproject.toml | 145 +++++++++++++++ .../otsu/src/otsu/__about__.py | 1 + .../otsu/src/otsu/__init__.py | 1 + .../command-line-tools/otsu/src/otsu/cli.py | 45 +++++ .../otsu/src/otsu/otsu_impl.py | 44 +++++ .../command-line-tools/otsu/tests/__init__.py | 0 .../otsu/tests/test_otsu.py | 52 ++++++ .../command-line-tools/stac/Dockerfile | 30 ++- .../command-line-tools/stac/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/stac/README.md | 50 +++++ water-bodies/command-line-tools/stac/app.py | 67 +------ .../command-line-tools/stac/pyproject.toml | 145 +++++++++++++++ .../stac/src/stac/__about__.py | 1 + .../stac/src/stac/__init__.py | 1 + .../command-line-tools/stac/src/stac/cli.py | 54 ++++++ .../stac/src/stac/stac_impl.py | 76 ++++++++ .../command-line-tools/stac/tests/__init__.py | 0 .../stac/tests/test_stac.py | 76 ++++++++ 53 files changed, 2604 insertions(+), 402 deletions(-) create mode 100644 tests/test_processing_pipeline.py create mode 100644 water-bodies/command-line-tools/crop/LICENSE.md create mode 100644 water-bodies/command-line-tools/crop/README.md create mode 100644 water-bodies/command-line-tools/crop/pyproject.toml create mode 100644 water-bodies/command-line-tools/crop/src/crop/__about__.py create mode 100644 water-bodies/command-line-tools/crop/src/crop/__init__.py create mode 100644 water-bodies/command-line-tools/crop/src/crop/cli.py create mode 100644 water-bodies/command-line-tools/crop/src/crop/crop_impl.py create mode 100644 water-bodies/command-line-tools/crop/tests/__init__.py create mode 100644 water-bodies/command-line-tools/crop/tests/test_crop_impl.py create mode 100644 water-bodies/command-line-tools/norm_diff/LICENSE.md create mode 100644 water-bodies/command-line-tools/norm_diff/README.md create mode 100644 water-bodies/command-line-tools/norm_diff/pyproject.toml create mode 100644 water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py create mode 100644 water-bodies/command-line-tools/norm_diff/src/norm_diff/__init__.py create mode 100644 water-bodies/command-line-tools/norm_diff/src/norm_diff/cli.py create mode 100644 water-bodies/command-line-tools/norm_diff/src/norm_diff/norm_diff_impl.py create mode 100644 water-bodies/command-line-tools/norm_diff/tests/__init__.py create mode 100644 water-bodies/command-line-tools/norm_diff/tests/test_norm_diff.py create mode 100644 water-bodies/command-line-tools/otsu/LICENSE.md create mode 100644 water-bodies/command-line-tools/otsu/README.md create mode 100644 water-bodies/command-line-tools/otsu/pyproject.toml create mode 100644 water-bodies/command-line-tools/otsu/src/otsu/__about__.py create mode 100644 water-bodies/command-line-tools/otsu/src/otsu/__init__.py create mode 100644 water-bodies/command-line-tools/otsu/src/otsu/cli.py create mode 100644 water-bodies/command-line-tools/otsu/src/otsu/otsu_impl.py create mode 100644 water-bodies/command-line-tools/otsu/tests/__init__.py create mode 100644 water-bodies/command-line-tools/otsu/tests/test_otsu.py create mode 100644 water-bodies/command-line-tools/stac/LICENSE.md create mode 100644 water-bodies/command-line-tools/stac/README.md create mode 100644 water-bodies/command-line-tools/stac/pyproject.toml create mode 100644 water-bodies/command-line-tools/stac/src/stac/__about__.py create mode 100644 water-bodies/command-line-tools/stac/src/stac/__init__.py create mode 100644 water-bodies/command-line-tools/stac/src/stac/cli.py create mode 100644 water-bodies/command-line-tools/stac/src/stac/stac_impl.py create mode 100644 water-bodies/command-line-tools/stac/tests/__init__.py create mode 100644 water-bodies/command-line-tools/stac/tests/test_stac.py diff --git a/cwl-cli/crop.cwl b/cwl-cli/crop.cwl index 74f2b67..08ce29c 100644 --- a/cwl-cli/crop.cwl +++ b/cwl-cli/crop.cwl @@ -35,9 +35,6 @@ class: CommandLineTool id: crop requirements: InlineJavascriptRequirement: {} - EnvVarRequirement: - envDef: - PYTHONPATH: /app ResourceRequirement: coresMax: 1 ramMax: 512 @@ -49,7 +46,7 @@ requirements: types: - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml -baseCommand: ["python", "-m", "app"] +baseCommand: crop arguments: [] inputs: item: @@ -63,13 +60,14 @@ inputs: type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi - valueFrom: $(self.bbox.join(",")) + valueFrom: $(JSON.stringify(self)) label: Area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. epsg: type: string inputBinding: prefix: --epsg + valueFrom: $(self.split(":").pop()) label: Coordinate reference system doc: EPSG code of the coordinate reference system used for the area of interest. band: diff --git a/cwl-cli/norm-diff.cwl b/cwl-cli/norm-diff.cwl index ccc4c7c..5642bd0 100644 --- a/cwl-cli/norm-diff.cwl +++ b/cwl-cli/norm-diff.cwl @@ -37,8 +37,7 @@ requirements: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMin: 2048 @@ -47,11 +46,15 @@ requirements: networkAccess: false DockerRequirement: dockerPull: localhost/norm-diff:latest -baseCommand: ["python", "-m", "app"] +baseCommand: norm_diff arguments: [] inputs: rasters: - type: File[] + type: + type: array + items: File + inputBinding: + prefix: --rasters inputBinding: position: 1 label: Input rasters diff --git a/cwl-cli/otsu.cwl b/cwl-cli/otsu.cwl index e7de87e..d583dc4 100644 --- a/cwl-cli/otsu.cwl +++ b/cwl-cli/otsu.cwl @@ -37,8 +37,7 @@ requirements: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /opt/conda/envs/env_otsu/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -46,13 +45,14 @@ requirements: networkAccess: false DockerRequirement: dockerPull: localhost/otsu:latest -baseCommand: ["python", "-m", "app"] +baseCommand: otsu arguments: [] inputs: raster: type: File inputBinding: position: 1 + prefix: --raster label: Water index raster doc: Normalized difference water index raster to threshold. outputs: diff --git a/cwl-cli/stac.cwl b/cwl-cli/stac.cwl index f5109cf..4e9b145 100644 --- a/cwl-cli/stac.cwl +++ b/cwl-cli/stac.cwl @@ -37,8 +37,7 @@ requirements: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -46,19 +45,19 @@ requirements: networkAccess: true DockerRequirement: dockerPull: localhost/stac:latest -baseCommand: ["python", "-m", "app"] +baseCommand: stac arguments: [] inputs: item: type: string inputBinding: - prefix: --input-item + prefix: --item label: STAC item doc: STAC item reference identifying the source acquisition. raster: type: File inputBinding: - prefix: --water-body + prefix: --rasters label: Water index raster doc: Normalized difference water index raster to threshold. outputs: diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index 92136ce..3c97254 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -48,7 +48,7 @@ $graph: inputs: aoi: label: area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code @@ -112,7 +112,7 @@ $graph: - $import: eoap-types.yaml inputs: aoi: - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon label: Area of interest epsg: @@ -175,8 +175,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -189,7 +188,7 @@ $graph: - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml - $import: eoap-types.yaml - baseCommand: ["python", "-m", "app"] + baseCommand: crop arguments: [] inputs: item: @@ -203,9 +202,9 @@ $graph: type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi - valueFrom: $(self.bbox.join(",")) + valueFrom: $(JSON.stringify(self)) label: Area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. epsg: type: 'eoap-types.yaml#EPSGCode' inputBinding: @@ -232,8 +231,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMin: 2048 @@ -242,11 +240,15 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/norm-diff:latest - baseCommand: ["python", "-m", "app"] + baseCommand: norm_diff arguments: [] inputs: rasters: - type: File[] + type: + type: array + items: File + inputBinding: + prefix: --rasters inputBinding: position: 1 label: Input rasters @@ -264,8 +266,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -273,13 +274,14 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/otsu:latest - baseCommand: ["python", "-m", "app"] + baseCommand: otsu arguments: [] inputs: raster: type: File inputBinding: position: 1 + prefix: --raster label: Water index raster doc: Normalized difference water index raster to threshold. outputs: @@ -295,8 +297,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -307,7 +308,7 @@ $graph: SchemaDefRequirement: types: - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml - baseCommand: ["python", "-m", "app"] + baseCommand: stac arguments: [] inputs: item: @@ -315,7 +316,7 @@ $graph: type: array items: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: - prefix: --input-item + prefix: --item valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. @@ -324,7 +325,7 @@ $graph: type: array items: File inputBinding: - prefix: --water-body + prefix: --rasters label: Input rasters doc: Raster files used for normalized difference computation or catalog generation. outputs: diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index 0d36704..a0987d5 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -47,7 +47,7 @@ $graph: inputs: aoi: label: area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code @@ -120,8 +120,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -134,7 +133,7 @@ $graph: - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml - $import: eoap-types.yaml - baseCommand: ["python", "-m", "app"] + baseCommand: crop arguments: [] inputs: item: @@ -148,9 +147,9 @@ $graph: type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi - valueFrom: $(self.bbox.join(",")) + valueFrom: $(JSON.stringify(self)) label: Area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. epsg: type: 'eoap-types.yaml#EPSGCode' inputBinding: @@ -177,8 +176,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMin: 2048 @@ -187,11 +185,15 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/norm-diff:latest - baseCommand: ["python", "-m", "app"] + baseCommand: norm_diff arguments: [] inputs: rasters: - type: File[] + type: + type: array + items: File + inputBinding: + prefix: --rasters inputBinding: position: 1 label: Input rasters @@ -209,8 +211,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -218,13 +219,14 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/otsu:latest - baseCommand: ["python", "-m", "app"] + baseCommand: otsu arguments: [] inputs: raster: type: File inputBinding: position: 1 + prefix: --raster label: Water index raster doc: Normalized difference water index raster to threshold. outputs: @@ -240,8 +242,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -252,20 +253,20 @@ $graph: SchemaDefRequirement: types: - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml - baseCommand: ["python", "-m", "app"] + baseCommand: stac arguments: [] inputs: item: type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI inputBinding: - prefix: --input-item + prefix: --item valueFrom: $(self.value) label: STAC item doc: STAC item reference identifying the source acquisition. rasters: type: File inputBinding: - prefix: --water-body + prefix: --rasters label: Input rasters doc: Raster files used for normalized difference computation or catalog generation. outputs: diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index bc06873..b9c4ccd 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -46,7 +46,7 @@ $graph: inputs: aoi: label: area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon epsg: label: EPSG code @@ -119,8 +119,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -132,7 +131,7 @@ $graph: types: - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml - $import: eoap-types.yaml - baseCommand: ["python", "-m", "app"] + baseCommand: crop arguments: [] inputs: item: @@ -145,9 +144,9 @@ $graph: type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi - valueFrom: $(self.bbox.join(",")) + valueFrom: $(JSON.stringify(self)) label: Area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. epsg: type: 'eoap-types.yaml#EPSGCode' inputBinding: @@ -174,8 +173,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMin: 2048 @@ -184,11 +182,15 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/norm-diff:latest - baseCommand: ["python", "-m", "app"] + baseCommand: norm_diff arguments: [] inputs: rasters: - type: File[] + type: + type: array + items: File + inputBinding: + prefix: --rasters inputBinding: position: 1 label: Input rasters @@ -206,8 +208,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -215,13 +216,14 @@ $graph: networkAccess: false DockerRequirement: dockerPull: localhost/otsu:latest - baseCommand: ["python", "-m", "app"] + baseCommand: otsu arguments: [] inputs: raster: type: File inputBinding: position: 1 + prefix: --raster label: Water index raster doc: Normalized difference water index raster to threshold. outputs: @@ -237,8 +239,7 @@ $graph: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /app + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 ramMax: 512 @@ -246,19 +247,19 @@ $graph: networkAccess: true DockerRequirement: dockerPull: localhost/stac:latest - baseCommand: ["python", "-m", "app"] + baseCommand: stac arguments: [] inputs: item: type: Directory inputBinding: - prefix: --input-item + prefix: --item label: STAC item doc: Directory containing the staged acquisition STAC catalog and assets. rasters: type: File inputBinding: - prefix: --water-body + prefix: --rasters label: Input rasters doc: Raster files used for normalized difference computation or catalog generation. outputs: diff --git a/docs/crop.cwl b/docs/crop.cwl index 8502d0f..1797906 100644 --- a/docs/crop.cwl +++ b/docs/crop.cwl @@ -37,7 +37,6 @@ requirements: InlineJavascriptRequirement: {} EnvVarRequirement: envDef: - PYTHONPATH: /app PROJ_LIB: /opt/conda/envs/env_crop/share/proj/ ResourceRequirement: coresMax: 1 @@ -48,7 +47,7 @@ requirements: types: - $import: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml -baseCommand: ["python", "-m", "app"] +baseCommand: crop arguments: [] inputs: item: @@ -62,13 +61,14 @@ inputs: type: https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon inputBinding: prefix: --aoi - valueFrom: $(self.bbox.join(",")) + valueFrom: $(JSON.stringify(self)) label: Area of interest - doc: GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. + doc: GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. epsg: type: string inputBinding: prefix: --epsg + valueFrom: $(self.split(":").pop()) label: Coordinate reference system doc: EPSG code of the coordinate reference system used for the area of interest. band: diff --git a/tests/test_processing_pipeline.py b/tests/test_processing_pipeline.py new file mode 100644 index 0000000..bc7a935 --- /dev/null +++ b/tests/test_processing_pipeline.py @@ -0,0 +1,46 @@ +"""Exercise the installed processing commands with local, synthetic STAC data.""" +import datetime +import json +import subprocess + +import numpy as np +import pystac +import rasterio +from rasterio.transform import from_origin + + +def test_processing_pipeline(tmp_path): + polygon = {'type': 'Polygon', 'coordinates': [[[1, 1], [9, 1], [1, 9], [1, 1]]]} + item = pystac.Item('synthetic', polygon, [1, 1, 9, 9], + datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), {}) + for band, pixels in [('green', np.full((10, 10), 100, dtype='uint16')), + ('nir', np.tile(np.arange(10, dtype='uint16') * 30 + 10, (10, 1)))]: + path = tmp_path / f'{band}.tif' + with rasterio.open(path, 'w', driver='GTiff', width=10, height=10, + count=1, dtype='uint16', crs='EPSG:4326', + transform=from_origin(0, 10, 1, 1), nodata=0) as dst: + dst.write(pixels, 1) + item.add_asset(band, pystac.Asset(str(path), roles=["data"], extra_fields={ + 'eo:bands': [{'name': band, 'common_name': band}]})) + item_path = tmp_path / 'source.json' + item.save_object(dest_href=str(item_path)) + def run(*args): + subprocess.run(args, cwd=tmp_path, check=True, capture_output=True, text=True) + for band in ['green', 'nir']: + run('crop', '--input-item', str(item_path), '--aoi', json.dumps(polygon), + '--epsg', '4326', '--band', band) + with rasterio.open(tmp_path / 'crop_green.tif') as src: + assert src.shape == (8, 8) + assert np.any(src.read(1) == 0), 'Polygon must mask pixels inside its bounding box' + assert np.any(src.read(1) == 100) + run('norm_diff', '--rasters', 'crop_green.tif', '--rasters', 'crop_nir.tif') + run('otsu', '--raster', 'norm_diff.tif') + run('stac', '--item', str(item_path), '--rasters', 'otsu.tif') + catalog = pystac.read_file(str(tmp_path / 'catalog.json')) + output = next(catalog.get_items(recursive=True)) + assert output.id == 'synthetic' + assert (tmp_path / 'synthetic' / 'otsu.tif').is_file() + assert output.assets['data'].href.lstrip('./') == 'otsu.tif' + with rasterio.open(output.assets['data'].get_absolute_href()) as src: + assert src.crs.to_epsg() == 4326 + assert set(np.unique(src.read(1))) <= {0, 1, 255} diff --git a/water-bodies/command-line-tools/crop/Dockerfile b/water-bodies/command-line-tools/crop/Dockerfile index a8c128c..57c1dad 100644 --- a/water-bodies/command-line-tools/crop/Dockerfile +++ b/water-bodies/command-line-tools/crop/Dockerfile @@ -1,13 +1,23 @@ -FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . -# System deps needed by rasterio wheels +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime RUN apt-get update && \ apt-get install -y --no-install-recommends libexpat1 && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir "rasterio==1.5.2" "click==8.5.0" "pystac==1.15.2" "loguru==0.7.3" "pyproj==3.8.0" "shapely==2.1.2" && \ - python -c "import rasterio" - -ADD app.py /app/app.py - -ENTRYPOINT [] \ No newline at end of file + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + crop --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work +ENTRYPOINT [] diff --git a/water-bodies/command-line-tools/crop/LICENSE.md b/water-bodies/command-line-tools/crop/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/crop/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/crop/README.md b/water-bodies/command-line-tools/crop/README.md new file mode 100644 index 0000000..141934e --- /dev/null +++ b/water-bodies/command-line-tools/crop/README.md @@ -0,0 +1,80 @@ +# crop + +Crop a STAC item's spectral-band asset to an area of interest and write a Cloud Optimized GeoTIFF (COG). + +## Set up with Hatch + +With Python 3.12 or newer and Hatch installed, run these commands from the repository root: + +```bash +cd water-bodies/command-line-tools/crop +hatch env create default +hatch run default:crop --help +``` + +Hatch installs this package in editable mode together with its dependencies. To run `crop` from another directory, activate the environment before changing directories: + +```bash +source "$(hatch env find default)/bin/activate" +mkdir -p /tmp/crop-results +cd /tmp/crop-results +``` + +Alternatively, install the package into an existing Python environment with `pip install -e .` from this package directory. + +## Crop a GeoJSON area + +Pass a GeoJSON geometry as a single quoted JSON string. This example uses a Polygon with coordinates in `[longitude, latitude]` order; the last coordinate repeats the first to close the ring. + +```bash +crop \ + --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ + --aoi '{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ + --epsg "4326" \ + --band green +``` + +This writes `crop_green.tif` in the current directory. To crop a near-infrared band, use `--band nir` or `--band nir08`, matching the common name in the item's asset metadata. + +All four options are required: + +| Option | Value | +| --- | --- | +| `--input-item` | STAC item URL or local path, or a staged catalog directory containing `catalog.json`. Catalog inputs use the first item found recursively. | +| `--aoi` | GeoJSON Polygon or MultiPolygon geometry, supplied as JSON text. Pass the geometry itself, not a Feature or a filename. | +| `--epsg` | EPSG code of the AOI. The generated CLI accepts `4326` (WGS 84). The AOI is transformed to the raster's CRS before cropping. | +| `--band` | Spectral common name: `green`, `nir`, or `nir08`. | + +The CLI also accepts the earlier bounding-box format: + +```bash +--aoi "-121.399,39.834,-120.74,40.472" +``` + +The four values are `xmin,ymin,xmax,ymax` in the AOI's coordinate reference system. The CWL interface uses a GeoJSON Polygon. The imported CWL schema also requires its `bbox` field. In `app-water-bodies-cloud-native.cwl`, use `epsg: "4326"`; STAC inputs are URI records such as `stac_items: [{value: "https://example.com/item.json"}]`. + +## Input and output behavior + +The selected asset must have a `data` role and a matching `common_name` in its `eo:bands` or `bands` metadata. Relative asset paths are resolved against the STAC item. Planetary Computer asset URLs are signed automatically when needed. + +The output is named `crop_.tif`, uses LZW compression, and retains the source raster's pixel datatype, CRS, and nodata value. Running the same band again in the same directory replaces that output. Remote inputs require network access. + +If you activated the environment manually, leave it with `deactivate` when finished. + +## Run tests + +From this package directory: + +```bash +hatch run test:test +``` + +The tests use local STAC items and synthetic rasters; they do not require remote imagery. + +## License + +`crop` is distributed under the [CC-BY-SA-4.0 license](LICENSE.md). + +## CLI source + +`src/crop/cli.py` defines the console interface. Processing logic lives in `crop_impl.py`. Update the CLI and CWL bindings together when changing arguments. diff --git a/water-bodies/command-line-tools/crop/app.py b/water-bodies/command-line-tools/crop/app.py index 98d8bb0..cd958f8 100644 --- a/water-bodies/command-line-tools/crop/app.py +++ b/water-bodies/command-line-tools/crop/app.py @@ -1,137 +1,5 @@ -import os -import time -import click -import pystac -import rasterio -from rasterio.mask import mask -from pyproj import Transformer -from shapely import box -from loguru import logger - - -def aoi2box(aoi): - """Converts an area of interest expressed as a bounding box to a list of floats""" - return [float(c) for c in aoi.split(",")] - - -def get_asset(item, common_name): - """Returns the asset of a STAC Item defined with its common band name""" - for _, asset in item.get_assets().items(): - if not "data" in asset.to_dict()["roles"]: - continue - - eo_asset = pystac.extensions.eo.AssetEOExtension(asset) - if not eo_asset.bands: - continue - for b in eo_asset.bands: - if ( - "common_name" in b.properties.keys() - and b.properties["common_name"] == common_name - ): - return asset - - -@click.command( - short_help="Crop", - help="Crops a STAC Item asset defined with its common band name", -) -@click.option( - "--input-item", - "item_url", - help="STAC Item URL or staged STAC catalog", - required=True, -) -@click.option( - "--aoi", - "aoi", - help="Area of interest expressed as a bounding box", - required=True, -) -@click.option( - "--epsg", - "epsg", - help="EPSG code", - required=True, -) -@click.option( - "--band", - "band", - help="Common band name", - required=True, -) -def crop(item_url, aoi, band, epsg): - - if os.path.isdir(item_url): - catalog = pystac.read_file(os.path.join(item_url, "catalog.json")) - item = next(catalog.get_items()) - else: - item = pystac.read_file(item_url) - - logger.info(f"Read {item.id} from {item.get_self_href()}") - - asset = get_asset(item, band) - logger.info(f"Read asset {band} from {asset.get_absolute_href()}") - - if not asset: - msg = f"Common band name {band} not found in the assets" - logger.error(msg) - raise ValueError(msg) - - bbox = aoi2box(aoi) - - for attempt in range(3): - try: - # Bound remote COG requests so a stalled connection cannot hang a lab. - with rasterio.Env( - GDAL_HTTP_CONNECTTIMEOUT=10, - GDAL_HTTP_TIMEOUT=60, - GDAL_HTTP_MAX_RETRY=2, - GDAL_HTTP_RETRY_DELAY=1, - GDAL_HTTP_RETRY_CODES="ALL", - GDAL_HTTP_MULTIRANGE="SERIAL", - CPL_VSIL_CURL_NON_CACHED=f"/vsicurl/{asset.get_absolute_href()}", - ), rasterio.open(asset.get_absolute_href()) as src: - - transformer = Transformer.from_crs(epsg, src.crs, always_xy=True) - - minx, miny = transformer.transform(bbox[0], bbox[1]) - maxx, maxy = transformer.transform(bbox[2], bbox[3]) - - transformed_bbox = box(minx, miny, maxx, maxy) - - logger.info(f"Crop {asset.get_absolute_href()}") - - out_image, out_transform = rasterio.mask.mask( - src, [transformed_bbox], crop=True - ) - out_meta = src.meta.copy() - - out_meta.update( - { - "height": out_image.shape[1], - "width": out_image.shape[2], - "transform": out_transform, - "dtype": "uint16", - "driver": "COG", - "tiled": True, - "compress": "lzw", - "blockxsize": 256, - "blockysize": 256, - } - ) - - with rasterio.open(f"crop_{band}.tif", "w", **out_meta) as dst_dataset: - logger.info(f"Write crop_{band}.tif") - dst_dataset.write(out_image) - break - except rasterio.errors.RasterioIOError: - if attempt == 2 or not asset.get_absolute_href().startswith(("http://", "https://")): - raise - logger.warning("Remote raster read failed; retrying crop ({}/3)", attempt + 2) - time.sleep(1) - - logger.info("Done!") - +"""Compatibility launcher; install the package and prefer `crop`.""" +from crop.cli import crop if __name__ == "__main__": crop() diff --git a/water-bodies/command-line-tools/crop/pyproject.toml b/water-bodies/command-line-tools/crop/pyproject.toml new file mode 100644 index 0000000..454dcc4 --- /dev/null +++ b/water-bodies/command-line-tools/crop/pyproject.toml @@ -0,0 +1,146 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "crop" +dynamic = ["version"] +description = "Crop satellite imagery to an area of interest" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "click==8.5.0", + "planetary-computer==1.0.0", + "pystac==1.15.2", + "rasterio==1.5.2", + "shapely==2.1.2", + "loguru==0.7.3" +] + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/crop/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/crop/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + +[project.scripts] +crop = "crop.cli:crop" + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/crop tests}" + +[tool.coverage.run] +source_pkgs = ["crop", "tests"] +branch = true +parallel = true +omit = [ + "src/crop/__about__.py", +] + +[tool.coverage.paths] +crop = ["src/crop", "*/crop/src/crop"] +tests = ["tests", "*/crop/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=crop --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff] +# Generated by cwl2click; regenerate rather than hand-edit this file. +extend-exclude = ["src/crop/cli.py"] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/crop/src/crop/__about__.py b/water-bodies/command-line-tools/crop/src/crop/__about__.py new file mode 100644 index 0000000..e5abda9 --- /dev/null +++ b/water-bodies/command-line-tools/crop/src/crop/__about__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2026-present U.N. Owen +# +# SPDX-License-Identifier: MIT +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/crop/src/crop/__init__.py b/water-bodies/command-line-tools/crop/src/crop/__init__.py new file mode 100644 index 0000000..deef26a --- /dev/null +++ b/water-bodies/command-line-tools/crop/src/crop/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026-present U.N. Owen +# +# SPDX-License-Identifier: MIT diff --git a/water-bodies/command-line-tools/crop/src/crop/cli.py b/water-bodies/command-line-tools/crop/src/crop/cli.py new file mode 100644 index 0000000..1f352fe --- /dev/null +++ b/water-bodies/command-line-tools/crop/src/crop/cli.py @@ -0,0 +1,72 @@ +# File generated by cwl2click v0.10.0 +# timestamp: 2026-09-23T15:58:25.186 + +__all__ = [] + +from crop.crop_impl import execute as crop_command + +from pathlib import Path + +import click + + + + + + + +# None +__all__.append("crop") +# NOTE +# Do not forget to add the section below in your `pyproject.toml` file +# +# [project.scripts] +# crop = "crop.cli:crop" + + +crop = click.Command( + name="crop", + callback=crop_command, + help="""Crop each requested spectral band to the area of interest.""", + short_help="""Crop Spectral Bands""", + params=[ + click.Option( + ["--input-item"], + "item", + type=click.STRING, + multiple=False, + required=True, + is_flag=False, + help="""URL of the source STAC item.""", + ), + click.Option( + ["--aoi"], + "aoi", + type=click.STRING, + multiple=False, + required=True, + is_flag=False, + help="""GeoJSON polygon defining the area of interest.""", + ), + click.Option( + ["--epsg"], + "epsg", + type=click.Choice(['4326']), + multiple=False, + required=True, + is_flag=False, + help="""Coordinate reference system of the area of interest.""", + ), + click.Option( + ["--band"], + "band", + type=click.Choice(['green', 'nir', 'nir08']), + multiple=False, + required=True, + is_flag=False, + help="""Common name of the spectral band to crop.""", + ), + ] +) + + diff --git a/water-bodies/command-line-tools/crop/src/crop/crop_impl.py b/water-bodies/command-line-tools/crop/src/crop/crop_impl.py new file mode 100644 index 0000000..e494fcc --- /dev/null +++ b/water-bodies/command-line-tools/crop/src/crop/crop_impl.py @@ -0,0 +1,157 @@ +"""Crop a STAC spectral-band asset to an area of interest.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import click +import planetary_computer +import pystac +import rasterio +from loguru import logger +from rasterio.mask import mask +from rasterio.warp import transform_geom +from shapely.geometry import Polygon, box, mapping, shape + + +def _read_item(location: str) -> pystac.Item: + path = Path(location) + source = str(path / "catalog.json") if path.is_dir() else location + logger.info("Reading input STAC document") + obj = pystac.read_file(source) + if isinstance(obj, pystac.Item): + logger.info("Loaded STAC item {}", obj.id) + return obj + if isinstance(obj, pystac.Catalog): + item = next(obj.get_items(recursive=True), None) + if item is not None: + logger.info("Selected first item {} from catalog {}", item.id, obj.id) + return item + raise ValueError("Input must be a STAC item or a catalog containing an item") + + +def _needs_signing(href: str) -> bool: + parsed = urlparse(href) + host = (parsed.hostname or "").lower() + return ( + host.endswith(".blob.core.windows.net") + and host != "ai4edatasetspublicassets.blob.core.windows.net" + and not {"st", "se", "sp"}.intersection(parse_qs(parsed.query)) + ) + + +def _bbox_geometry(aoi: str) -> Polygon: + """Parse and validate a comma-separated bounding box.""" + bounds = [float(value) for value in aoi.split(",")] + if len(bounds) != 4: + raise ValueError("AOI must be a GeoJSON polygon or xmin,ymin,xmax,ymax") + if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]: + raise ValueError("AOI minimum coordinates must be smaller than maxima") + return box(bounds[0], bounds[1], bounds[2], bounds[3]) + + +def _geometry(aoi: str) -> dict: + geometry = ( + shape(json.loads(aoi)) if aoi.lstrip().startswith("{") else _bbox_geometry(aoi) + ) + if geometry.geom_type not in {"Polygon", "MultiPolygon"}: + raise ValueError("AOI must be a polygon") + if geometry.is_empty or not geometry.is_valid: + raise ValueError("AOI must be a nonempty, valid polygon") + return mapping(geometry) + + +def _asset(item: pystac.Item, common_name: str) -> pystac.Asset: + for key, asset in item.assets.items(): + if "data" not in (asset.roles or []): + continue + bands = asset.extra_fields.get("eo:bands", []) + bands = bands or asset.extra_fields.get("bands", []) + if any(band.get("common_name") == common_name for band in bands): + logger.info("Selected asset {} for band {}", key, common_name) + return asset + raise ValueError(f"Common band name {common_name} not found in the assets") + + +def _asset_href(item: pystac.Item, band: str) -> str: + """Sign cloud assets when needed and resolve the selected band's location.""" + if any(_needs_signing(asset.href) for asset in item.assets.values()): + logger.info("Signing Planetary Computer asset URLs") + planetary_computer.sign_inplace(item) + asset = _asset(item, band) + href = asset.get_absolute_href() + if href is None: + raise ValueError("Selected asset has no resolvable href") + return href + + +def execute(*, input_item: str, aoi: str, epsg: str, band: str) -> None: + """Read the CLI inputs and write ``crop_.tif`` in the working directory.""" + try: + logger.info("Starting crop for band {}", band) + if band not in {"green", "nir", "nir08"}: + raise ValueError("Band must be one of green, nir, nir08") + epsg = f"EPSG:{epsg}" if epsg.isdigit() else epsg + geometry = _geometry(aoi) + logger.info("Parsed {} AOI in {}", geometry["type"], epsg) + item = _read_item(input_item) + href = _asset_href(item, band) + logger.info("Opening raster for item {}, band {}", item.id, band) + for attempt in range(3): + try: + with rasterio.Env( + GDAL_HTTP_CONNECTTIMEOUT=10, + GDAL_HTTP_TIMEOUT=60, + GDAL_HTTP_MAX_RETRY=2, + GDAL_HTTP_RETRY_DELAY=1, + GDAL_HTTP_RETRY_CODES="ALL", + GDAL_HTTP_MULTIRANGE="SERIAL", + CPL_VSIL_CURL_NON_CACHED=f"/vsicurl/{href}", + ), rasterio.open(href) as src: + if src.crs is None: + raise ValueError("Selected raster has no coordinate reference system") + logger.info( + "Source raster: {} x {} pixels, {} band(s), CRS {}", + src.width, + src.height, + src.count, + src.crs, + ) + logger.info("Transforming AOI from {} to {}", epsg, src.crs) + projected = transform_geom(epsg, src.crs, geometry) + logger.info("Cropping raster to the AOI") + pixels, transform = mask(src, [projected], crop=True) + logger.info( + "Cropped raster: {} x {} pixels", pixels.shape[2], pixels.shape[1] + ) + metadata = src.meta.copy() + metadata.update( + driver="COG", + height=pixels.shape[1], + width=pixels.shape[2], + transform=transform, + compress="LZW", + blocksize=256, + ) + output = Path(f"crop_{band}.tif") + logger.info("Writing LZW-compressed COG to {}", output.resolve()) + with rasterio.open(output, "w", **metadata) as dst: + dst.write(pixels) + break + except rasterio.errors.RasterioIOError: + if attempt == 2 or not href.startswith(("http://", "https://")): + raise + logger.warning("Remote raster read failed; retrying crop ({}/3)", attempt + 2) + time.sleep(1) + logger.success("Crop complete: {}", output.resolve()) + except ( + ValueError, + OSError, + pystac.STACError, + rasterio.errors.RasterioError, + ) as exc: + logger.error("Crop failed: {}", exc) + raise click.ClickException(str(exc)) from exc diff --git a/water-bodies/command-line-tools/crop/tests/__init__.py b/water-bodies/command-line-tools/crop/tests/__init__.py new file mode 100644 index 0000000..deef26a --- /dev/null +++ b/water-bodies/command-line-tools/crop/tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2026-present U.N. Owen +# +# SPDX-License-Identifier: MIT diff --git a/water-bodies/command-line-tools/crop/tests/test_crop_impl.py b/water-bodies/command-line-tools/crop/tests/test_crop_impl.py new file mode 100644 index 0000000..976aaae --- /dev/null +++ b/water-bodies/command-line-tools/crop/tests/test_crop_impl.py @@ -0,0 +1,148 @@ +"""Exercise the unchanged generated CLI with real local STAC and raster inputs.""" + +import json +from datetime import datetime, timezone + +import numpy as np +import pystac +import pytest +import rasterio +from click.testing import CliRunner +from rasterio.transform import from_origin + +from crop.cli import crop +from crop.crop_impl import _needs_signing + + +@pytest.fixture +def source(tmp_path): + pixels = np.arange(100, dtype="float32").reshape(10, 10) + with rasterio.open( + tmp_path / "source.tif", + "w", + driver="GTiff", + width=10, + height=10, + count=1, + dtype="float32", + crs="EPSG:4326", + transform=from_origin(0, 10, 1, 1), + nodata=-9999, + ) as dst: + dst.write(pixels, 1) + item = pystac.Item( + "source", + geometry={ + "type": "Polygon", + "coordinates": [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]], + }, + bbox=[0, 0, 10, 10], + datetime=datetime.now(timezone.utc), + properties={}, + ) + item.add_asset( + "green", + pystac.Asset( + "./source.tif", + roles=["data"], + extra_fields={"eo:bands": [{"common_name": "green"}]}, + ), + ) + item.set_self_href(str(tmp_path / "item.json")) + item.save_object() + return item, pixels + + +@pytest.mark.parametrize("geojson", [False, True]) +def test_cli_crops_local_asset(source, tmp_path, monkeypatch, geojson): + item, pixels = source + monkeypatch.chdir(tmp_path) + aoi = ( + json.dumps( + { + "type": "Polygon", + "coordinates": [[[2, 4], [5, 4], [5, 8], [2, 8], [2, 4]]], + } + ) + if geojson + else "2,4,5,8" + ) + result = CliRunner().invoke( + crop, + [ + "--input-item", + item.get_self_href(), + "--aoi", + aoi, + "--epsg", + "4326", + "--band", + "green", + ], + ) + assert result.exit_code == 0, result.output + with rasterio.open(tmp_path / "crop_green.tif") as output: + np.testing.assert_array_equal(output.read(1), pixels[2:6, 2:5]) + assert output.transform == from_origin(2, 8, 1, 1) + assert output.dtypes == ("float32",) + assert output.nodata == -9999 + assert output.tags(ns="IMAGE_STRUCTURE")["LAYOUT"] == "COG" + + +def test_catalog_input(source, tmp_path, monkeypatch): + item, _ = source + catalog = pystac.Catalog("staged", "Staged input") + catalog.add_item(item) + catalog.set_self_href(str(tmp_path / "catalog.json")) + catalog.save_object() + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke( + crop, + [ + "--input-item", + str(tmp_path), + "--aoi", + "2,4,5,8", + "--epsg", + "4326", + "--band", + "green", + ], + ) + assert result.exit_code == 0, result.output + assert (tmp_path / "crop_green.tif").exists() + + +def test_missing_band_reports_error(source, tmp_path, monkeypatch): + item, _ = source + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke( + crop, + [ + "--input-item", + item.get_self_href(), + "--aoi", + "2,4,5,8", + "--epsg", + "4326", + "--band", + "nir", + ], + ) + assert result.exit_code == 1 + assert "Common band name nir not found" in result.output + assert not (tmp_path / "crop_nir.tif").exists() + + +@pytest.mark.parametrize( + "href, expected", + [ + ("https://example.blob.core.windows.net/a.tif", True), + ("https://example.blob.core.windows.net/a.tif?se=tomorrow", False), + ("https://ai4edatasetspublicassets.blob.core.windows.net/a.tif", False), + ("https://example.org/a.tif", False), + ("./source.tif", False), + ], +) +def test_signing_detection(href, expected): + assert _needs_signing(href) is expected diff --git a/water-bodies/command-line-tools/norm_diff/Dockerfile b/water-bodies/command-line-tools/norm_diff/Dockerfile index 8cce651..904e97e 100644 --- a/water-bodies/command-line-tools/norm_diff/Dockerfile +++ b/water-bodies/command-line-tools/norm_diff/Dockerfile @@ -1,14 +1,23 @@ -FROM docker.io/library/python:3.11-slim@sha256:91e9d01cf4bd56be7128c603506b6fe367ef7506f9f2ad8f3a908aeec8941bb9 +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . -# System deps needed by rasterio wheels +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime RUN apt-get update && \ apt-get install -y --no-install-recommends libexpat1 && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir "rasterio==1.4.3" "click==8.2.1" "loguru==0.7.3" && \ - python -c "import rasterio" - -ADD app.py /app/app.py - + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + norm_diff --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work ENTRYPOINT [] - diff --git a/water-bodies/command-line-tools/norm_diff/LICENSE.md b/water-bodies/command-line-tools/norm_diff/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/norm_diff/README.md b/water-bodies/command-line-tools/norm_diff/README.md new file mode 100644 index 0000000..a9fe348 --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/README.md @@ -0,0 +1,50 @@ +# norm_diff + +Writes `norm_diff.tif` as a float32 COG using `(first - second) / (first + second)`. Inputs must share a raster grid. Zero denominators retain NumPy NaN/infinity behavior. + +## Hatch setup + +From the repository root: + +```bash +cd water-bodies/command-line-tools/norm_diff +hatch env create default +source "$(hatch env find default)/bin/activate" +``` + +Python 3.12 or newer and Hatch are required. Run the command from the directory where outputs should be written: + +```bash +norm_diff --rasters green.tif --rasters nir.tif +``` + +Run `deactivate` when finished. Progress and failures are logged by Loguru for the local processing tools. + +## Checks and tests + +From this package directory: + +```bash +hatch run test:test +``` + +The Ruff complexity limit is 4. Tests use local fixtures rather than remote imagery. + +## Container + +From the repository root: + +```bash +podman build -t localhost:norm-diff:latest water-bodies/command-line-tools/norm_diff +``` + +The Dockerfile builds a wheel with Hatchling through pip and installs it in a non-root runtime environment. +Invoke the installed console command explicitly, for example `podman run --rm localhost/norm-diff:latest norm_diff --help`. + +## License + +See [LICENSE.md](LICENSE.md). + +## CLI source + +`src/norm_diff/cli.py` defines the console interface. Processing logic lives in `norm_diff_impl.py`. Update the CLI and CWL bindings together when changing arguments. diff --git a/water-bodies/command-line-tools/norm_diff/app.py b/water-bodies/command-line-tools/norm_diff/app.py index 78b0bcf..6dbbd06 100644 --- a/water-bodies/command-line-tools/norm_diff/app.py +++ b/water-bodies/command-line-tools/norm_diff/app.py @@ -1,46 +1,5 @@ -"""Normalized difference""" -import click -import rasterio -import numpy as np -from loguru import logger - -np.seterr(divide="ignore", invalid="ignore") - - -@click.command( - short_help="Normalized difference", - help="Performs a normalized difference", -) -@click.argument("rasters", nargs=2) -def normalized_difference(rasters): - """Performs a normalized difference""" - - logger.info(f"Processing the normalized image with {rasters[0]} and {rasters[1]}") - - with rasterio.open(rasters[0]) as ds1: - array1 = ds1.read(1).astype(np.float32) - out_meta = ds1.meta.copy() - - with rasterio.open(rasters[1]) as ds2: - array2 = ds2.read(1).astype(np.float32) - - out_meta.update( - { - "dtype": "float32", - "driver": "COG", - "tiled": True, - "compress": "lzw", - "blockxsize": 256, - "blockysize": 256, - } - ) - - with rasterio.open("norm_diff.tif", "w", **out_meta) as dst_dataset: - logger.info(f"Write norm_diff.tif") - dst_dataset.write((array1 - array2) / (array1 + array2), 1) - - logger.info("Done!") - +"""Compatibility launcher; install the package and prefer `norm_diff`.""" +from norm_diff.cli import norm_diff if __name__ == "__main__": - normalized_difference() + norm_diff() diff --git a/water-bodies/command-line-tools/norm_diff/pyproject.toml b/water-bodies/command-line-tools/norm_diff/pyproject.toml new file mode 100644 index 0000000..f347d61 --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/pyproject.toml @@ -0,0 +1,144 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "norm_diff" +dynamic = ["version"] +description = "Calculate normalized difference from two rasters" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "click==8.5.0", + "rasterio==1.5.2", + "numpy>=1.24", + "loguru==0.7.3", +] + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/norm_diff/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/norm_diff/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + +[project.scripts] +norm_diff = "norm_diff.cli:norm_diff" + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/norm_diff tests}" + +[tool.coverage.run] +source_pkgs = ["norm_diff", "tests"] +branch = true +parallel = true +omit = [ + "src/norm_diff/__about__.py", +] + +[tool.coverage.paths] +norm_diff = ["src/norm_diff", "*/norm_diff/src/norm_diff"] +tests = ["tests", "*/norm_diff/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=norm_diff --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff] +# Generated by cwl2click; regenerate rather than hand-edit this file. +extend-exclude = ["src/norm_diff/cli.py"] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/norm_diff/src/norm_diff/__init__.py b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__init__.py new file mode 100644 index 0000000..f08a946 --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__init__.py @@ -0,0 +1 @@ +"""Calculate normalized difference from two rasters.""" diff --git a/water-bodies/command-line-tools/norm_diff/src/norm_diff/cli.py b/water-bodies/command-line-tools/norm_diff/src/norm_diff/cli.py new file mode 100644 index 0000000..9f252e7 --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/src/norm_diff/cli.py @@ -0,0 +1,45 @@ +# File generated by cwl2click v0.10.0 +# timestamp: 2026-09-23T15:58:29.884 + +__all__ = [] + +from norm_diff.norm_diff_impl import execute as norm_diff_command + +from pathlib import Path + +import click + + + + + + + +# None +__all__.append("norm_diff") +# NOTE +# Do not forget to add the section below in your `pyproject.toml` file +# +# [project.scripts] +# norm_diff = "norm_diff.cli:norm_diff" + + +norm_diff = click.Command( + name="norm_diff", + callback=norm_diff_command, + help="""Calculate the normalized difference water index (NDWI) from the green and near-infrared spectral band rasters.""", + short_help="""Normalized Difference Calculation""", + params=[ + click.Option( + ["--rasters"], + "rasters", + type=click.Path(path_type=Path, exists=True, readable=True, resolve_path=True, file_okay=True, dir_okay=False), + multiple=True, + required=True, + is_flag=False, + help="""Ordered green and near-infrared GeoTIFFs used to calculate NDWI.""", + ), + ] +) + + diff --git a/water-bodies/command-line-tools/norm_diff/src/norm_diff/norm_diff_impl.py b/water-bodies/command-line-tools/norm_diff/src/norm_diff/norm_diff_impl.py new file mode 100644 index 0000000..8038e4b --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/src/norm_diff/norm_diff_impl.py @@ -0,0 +1,43 @@ +"""Normalized difference raster processing.""" + +from pathlib import Path + +import click +import numpy as np +import rasterio +from loguru import logger + + +def _calculate(rasters: tuple[str | Path, ...]) -> tuple[np.ndarray, dict]: + if len(rasters) != 2: + raise ValueError("Exactly two rasters are required") + with rasterio.open(rasters[0]) as first, rasterio.open(rasters[1]) as second: + grid1 = (first.shape, first.transform, first.crs) + grid2 = (second.shape, second.transform, second.crs) + if grid1 != grid2: + raise ValueError( + "Input rasters must have matching dimensions, transform, and CRS" + ) + a = first.read(1).astype(np.float32) + b = second.read(1).astype(np.float32) + metadata = first.meta.copy() + with np.errstate(divide="ignore", invalid="ignore"): + result = (a - b) / (a + b) + return result, metadata + + +def execute(*, rasters: tuple[str | Path, ...]) -> None: + """Write the normalized difference to norm_diff.tif.""" + try: + logger.info("Calculating normalized difference from {}", rasters) + data, metadata = _calculate(rasters) + metadata.update( + driver="COG", dtype="float32", count=1, compress="LZW", blocksize=256 + ) + logger.info("Writing norm_diff.tif") + with rasterio.open("norm_diff.tif", "w", **metadata) as output: + output.write(data, 1) + logger.success("Wrote norm_diff.tif") + except (ValueError, OSError, rasterio.errors.RasterioError) as exc: + logger.error("Normalized difference failed: {}", exc) + raise click.ClickException(str(exc)) from exc diff --git a/water-bodies/command-line-tools/norm_diff/tests/__init__.py b/water-bodies/command-line-tools/norm_diff/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/water-bodies/command-line-tools/norm_diff/tests/test_norm_diff.py b/water-bodies/command-line-tools/norm_diff/tests/test_norm_diff.py new file mode 100644 index 0000000..34d453e --- /dev/null +++ b/water-bodies/command-line-tools/norm_diff/tests/test_norm_diff.py @@ -0,0 +1,68 @@ +"""Exercise raster processing with local fixtures.""" + +import numpy as np +import pytest +import rasterio +from click.testing import CliRunner +from rasterio.transform import from_origin + +from norm_diff.cli import norm_diff + + +@pytest.fixture +def raster_factory(tmp_path): + def write(name, data): + path = tmp_path / name + data = np.asarray(data, dtype="float32") + with rasterio.open( + path, + "w", + driver="GTiff", + width=data.shape[1], + height=data.shape[0], + count=1, + dtype="float32", + crs="EPSG:4326", + transform=from_origin(0, 2, 1, 1), + ) as dst: + dst.write(data, 1) + return str(path) + + return write + + +def test_normalized_difference(raster_factory, tmp_path, monkeypatch): + a = raster_factory("a.tif", [[3, 0], [1, 4]]) + b = raster_factory("b.tif", [[1, 0], [3, 4]]) + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(norm_diff, ["--rasters", a, "--rasters", b]) + assert result.exit_code == 0, result.output + with rasterio.open("norm_diff.tif") as output: + np.testing.assert_allclose( + output.read(1), [[0.5, np.nan], [-0.5, 0]], equal_nan=True + ) + assert output.tags(ns="IMAGE_STRUCTURE")["LAYOUT"] == "COG" + assert output.count == 1 + + +def test_mismatched_grid(raster_factory, tmp_path, monkeypatch): + a = raster_factory("a.tif", [[1, 2]]) + b = raster_factory("b.tif", [[1], [2]]) + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(norm_diff, ["--rasters", a, "--rasters", b]) + assert result.exit_code == 1 + assert "matching dimensions" in result.output + assert not (tmp_path / "norm_diff.tif").exists() + + +def test_missing_rasters(): + result = CliRunner().invoke(norm_diff, []) + assert result.exit_code == 2 + assert "Missing option" in result.output + + +def test_single_raster(raster_factory): + raster = raster_factory("one.tif", [[1, 2]]) + result = CliRunner().invoke(norm_diff, ["--rasters", raster]) + assert result.exit_code == 1 + assert "Exactly two" in result.output diff --git a/water-bodies/command-line-tools/otsu/Dockerfile b/water-bodies/command-line-tools/otsu/Dockerfile index ac145d3..713e717 100644 --- a/water-bodies/command-line-tools/otsu/Dockerfile +++ b/water-bodies/command-line-tools/otsu/Dockerfile @@ -1,13 +1,23 @@ -FROM docker.io/library/python:3.11-slim@sha256:91e9d01cf4bd56be7128c603506b6fe367ef7506f9f2ad8f3a908aeec8941bb9 +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . -# System deps needed by rasterio wheels +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime RUN apt-get update && \ apt-get install -y --no-install-recommends libexpat1 && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir "rasterio==1.4.3" "scikit-image==0.25.2" "click==8.2.1" "loguru==0.7.3" && \ - python -c "import rasterio" - -ADD app.py /app/app.py - + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + otsu --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work ENTRYPOINT [] diff --git a/water-bodies/command-line-tools/otsu/LICENSE.md b/water-bodies/command-line-tools/otsu/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/otsu/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/otsu/README.md b/water-bodies/command-line-tools/otsu/README.md new file mode 100644 index 0000000..fbe3123 --- /dev/null +++ b/water-bodies/command-line-tools/otsu/README.md @@ -0,0 +1,50 @@ +# otsu + +Writes `otsu.tif` as a uint8 COG with 0 for non-water and 1 for water. The threshold is computed from finite values; nonfinite pixels are marked 0. An input with no finite values is rejected. + +## Hatch setup + +From the repository root: + +```bash +cd water-bodies/command-line-tools/otsu +hatch env create default +source "$(hatch env find default)/bin/activate" +``` + +Python 3.12 or newer and Hatch are required. Run the command from the directory where outputs should be written: + +```bash +otsu --raster norm_diff.tif +``` + +Run `deactivate` when finished. Progress and failures are logged by Loguru for the local processing tools. + +## Checks and tests + +From this package directory: + +```bash +hatch run test:test +``` + +The Ruff complexity limit is 4. Tests use local fixtures rather than remote imagery. + +## Container + +From the repository root: + +```bash +podman build -t localhost:otsu:latest water-bodies/command-line-tools/otsu +``` + +The Dockerfile builds a wheel with Hatchling through pip and installs it in a non-root runtime environment. +Invoke the installed console command explicitly, for example `podman run --rm localhost/otsu:latest otsu --help`. + +## License + +See [LICENSE.md](LICENSE.md). + +## CLI source + +`src/otsu/cli.py` defines the console interface. Processing logic lives in `otsu_impl.py`. Update the CLI and CWL bindings together when changing arguments. diff --git a/water-bodies/command-line-tools/otsu/app.py b/water-bodies/command-line-tools/otsu/app.py index f4e7b6c..de29895 100644 --- a/water-bodies/command-line-tools/otsu/app.py +++ b/water-bodies/command-line-tools/otsu/app.py @@ -1,47 +1,5 @@ -"""Command line tool to apply the Otsu threshold to a raster""" -import click -import rasterio -import numpy as np -from skimage.filters import threshold_otsu -from loguru import logger - - -def threshold(data): - """Returns the Otsu threshold of a numpy array""" - return data > threshold_otsu(data[np.isfinite(data)]) - - -@click.command( - short_help="Otsu threshoold", - help="Applies the Otsu threshold", -) -@click.argument("raster", nargs=1) -def otsu(raster): - """Applies the Otsu threshold""" - - with rasterio.open(raster) as ds: - array = ds.read(1) - out_meta = ds.meta.copy() - - out_meta.update( - { - "dtype": "uint8", - "driver": "COG", - "tiled": True, - "compress": "lzw", - "blockxsize": 256, - "blockysize": 256, - } - ) - - logger.info(f"Applying the Otsu threshold to {raster}") - - with rasterio.open("otsu.tif", "w", **out_meta) as dst_dataset: - logger.info(f"Write otsu.tif") - dst_dataset.write(threshold(array), indexes=1) - - logger.info("Done!") - +"""Compatibility launcher; install the package and prefer `otsu`.""" +from otsu.cli import otsu if __name__ == "__main__": otsu() diff --git a/water-bodies/command-line-tools/otsu/pyproject.toml b/water-bodies/command-line-tools/otsu/pyproject.toml new file mode 100644 index 0000000..9e5577a --- /dev/null +++ b/water-bodies/command-line-tools/otsu/pyproject.toml @@ -0,0 +1,145 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "otsu" +dynamic = ["version"] +description = "Apply Otsu thresholding to a raster" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "click==8.5.0", + "rasterio==1.5.2", + "numpy>=1.24", + "scikit-image==0.26.0", + "loguru==0.7.3", +] + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/otsu/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/otsu/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + +[project.scripts] +otsu = "otsu.cli:otsu" + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/otsu tests}" + +[tool.coverage.run] +source_pkgs = ["otsu", "tests"] +branch = true +parallel = true +omit = [ + "src/otsu/__about__.py", +] + +[tool.coverage.paths] +otsu = ["src/otsu", "*/otsu/src/otsu"] +tests = ["tests", "*/otsu/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=otsu --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff] +# Generated by cwl2click; regenerate rather than hand-edit this file. +extend-exclude = ["src/otsu/cli.py"] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/otsu/src/otsu/__about__.py b/water-bodies/command-line-tools/otsu/src/otsu/__about__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/water-bodies/command-line-tools/otsu/src/otsu/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/otsu/src/otsu/__init__.py b/water-bodies/command-line-tools/otsu/src/otsu/__init__.py new file mode 100644 index 0000000..82a42f7 --- /dev/null +++ b/water-bodies/command-line-tools/otsu/src/otsu/__init__.py @@ -0,0 +1 @@ +"""Apply Otsu thresholding to a raster.""" diff --git a/water-bodies/command-line-tools/otsu/src/otsu/cli.py b/water-bodies/command-line-tools/otsu/src/otsu/cli.py new file mode 100644 index 0000000..c3f6e8e --- /dev/null +++ b/water-bodies/command-line-tools/otsu/src/otsu/cli.py @@ -0,0 +1,45 @@ +# File generated by cwl2click v0.10.0 +# timestamp: 2026-09-23T15:58:36.669 + +__all__ = [] + +from otsu.otsu_impl import execute as otsu_command + +from pathlib import Path + +import click + + + + + + + +# None +__all__.append("otsu") +# NOTE +# Do not forget to add the section below in your `pyproject.toml` file +# +# [project.scripts] +# otsu = "otsu.cli:otsu" + + +otsu = click.Command( + name="otsu", + callback=otsu_command, + help="""Apply Otsu thresholding to the NDWI raster to generate a binary water body mask.""", + short_help="""Otsu Thresholding""", + params=[ + click.Option( + ["--raster"], + "raster", + type=click.Path(path_type=Path, exists=True, readable=True, resolve_path=True, file_okay=True, dir_okay=False), + multiple=False, + required=True, + is_flag=False, + help="""Normalized difference water index raster to threshold.""", + ), + ] +) + + diff --git a/water-bodies/command-line-tools/otsu/src/otsu/otsu_impl.py b/water-bodies/command-line-tools/otsu/src/otsu/otsu_impl.py new file mode 100644 index 0000000..536ded5 --- /dev/null +++ b/water-bodies/command-line-tools/otsu/src/otsu/otsu_impl.py @@ -0,0 +1,44 @@ +"""Otsu water mask processing.""" + +from pathlib import Path + +import click +import numpy as np +import rasterio +from loguru import logger +from skimage.filters import threshold_otsu + + +def threshold(data: np.ndarray) -> np.ndarray: + """Threshold finite raster values; mark nonfinite pixels as non-water.""" + finite = np.isfinite(data) + if not finite.any(): + raise ValueError("Raster has no finite values for Otsu thresholding") + cutoff = threshold_otsu(data[finite]) + logger.info("Otsu threshold: {}", cutoff) + return (finite & (data > cutoff)).astype(np.uint8) + + +def execute(*, raster: str | Path) -> None: + """Write an unsigned-byte binary mask to otsu.tif.""" + try: + logger.info("Reading {}", raster) + with rasterio.open(raster) as source: + data = source.read(1) + metadata = source.meta.copy() + result = threshold(data) + metadata.update( + driver="COG", + dtype="uint8", + count=1, + nodata=None, + compress="LZW", + blocksize=256, + ) + logger.info("Writing otsu.tif") + with rasterio.open("otsu.tif", "w", **metadata) as output: + output.write(result, 1) + logger.success("Wrote otsu.tif") + except (ValueError, OSError, rasterio.errors.RasterioError) as exc: + logger.error("Otsu thresholding failed: {}", exc) + raise click.ClickException(str(exc)) from exc diff --git a/water-bodies/command-line-tools/otsu/tests/__init__.py b/water-bodies/command-line-tools/otsu/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/water-bodies/command-line-tools/otsu/tests/test_otsu.py b/water-bodies/command-line-tools/otsu/tests/test_otsu.py new file mode 100644 index 0000000..3d5750f --- /dev/null +++ b/water-bodies/command-line-tools/otsu/tests/test_otsu.py @@ -0,0 +1,52 @@ +"""Exercise raster processing with local fixtures.""" + +import numpy as np +import pytest +import rasterio +from click.testing import CliRunner +from rasterio.transform import from_origin + +from otsu.cli import otsu + + +@pytest.fixture +def raster_factory(tmp_path): + def write(name, data): + path = tmp_path / name + data = np.asarray(data, dtype="float32") + with rasterio.open( + path, + "w", + driver="GTiff", + width=data.shape[1], + height=data.shape[0], + count=1, + dtype="float32", + crs="EPSG:4326", + transform=from_origin(0, 2, 1, 1), + ) as dst: + dst.write(data, 1) + return str(path) + + return write + + +def test_binary_mask(raster_factory, tmp_path, monkeypatch): + raster = raster_factory("input.tif", [[0, 0, np.nan], [10, 10, np.inf]]) + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(otsu, ["--raster", raster]) + assert result.exit_code == 0, result.output + with rasterio.open("otsu.tif") as output: + np.testing.assert_array_equal(output.read(1), [[0, 0, 0], [1, 1, 0]]) + assert output.dtypes == ("uint8",) + assert output.nodata is None + assert output.tags(ns="IMAGE_STRUCTURE")["LAYOUT"] == "COG" + + +def test_no_finite_values(raster_factory, tmp_path, monkeypatch): + raster = raster_factory("invalid.tif", [[np.nan, np.inf]]) + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(otsu, ["--raster", raster]) + assert result.exit_code == 1 + assert "no finite values" in result.output + assert not (tmp_path / "otsu.tif").exists() diff --git a/water-bodies/command-line-tools/stac/Dockerfile b/water-bodies/command-line-tools/stac/Dockerfile index 1246009..8274ef1 100644 --- a/water-bodies/command-line-tools/stac/Dockerfile +++ b/water-bodies/command-line-tools/stac/Dockerfile @@ -1,13 +1,23 @@ -FROM docker.io/library/python:3.11-slim@sha256:91e9d01cf4bd56be7128c603506b6fe367ef7506f9f2ad8f3a908aeec8941bb9 +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . -# System deps needed by rasterio wheels +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime RUN apt-get update && \ apt-get install -y --no-install-recommends libexpat1 && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir "pystac==1.13.0" "rio-stac==0.11.2" "loguru==0.7.3" "click==8.2.1" && \ - python -c "import rio_stac" - -ADD app.py /app/app.py - -ENTRYPOINT [] \ No newline at end of file + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + stac --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work +ENTRYPOINT [] diff --git a/water-bodies/command-line-tools/stac/LICENSE.md b/water-bodies/command-line-tools/stac/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/stac/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/stac/README.md b/water-bodies/command-line-tools/stac/README.md new file mode 100644 index 0000000..2cc53f6 --- /dev/null +++ b/water-bodies/command-line-tools/stac/README.md @@ -0,0 +1,50 @@ +# stac + +Writes `catalog.json` and an item directory containing each mask. Repeat both options for multiple pairs in corresponding order. Inputs may be STAC item URLs, local item files, or staged catalog directories. Item IDs must be unique directory names. + +## Hatch setup + +From the repository root: + +```bash +cd water-bodies/command-line-tools/stac +hatch env create default +source "$(hatch env find default)/bin/activate" +``` + +Python 3.12 or newer and Hatch are required. Run the command from the directory where outputs should be written: + +```bash +stac --item source-item.json --rasters otsu.tif +``` + +Run `deactivate` when finished. Progress and failures are logged by Loguru for the local processing tools. + +## Checks and tests + +From this package directory: + +```bash +hatch run test:test +``` + +The Ruff complexity limit is 4. Tests use local fixtures rather than remote imagery. + +## Container + +From the repository root: + +```bash +podman build -t localhost/stac:latest water-bodies/command-line-tools/stac +``` + +The Dockerfile builds a wheel with Hatchling through pip and installs it in a non-root runtime environment. +Invoke the installed console command explicitly, for example `podman run --rm localhost/stac:latest stac --help`. + +## License + +See [LICENSE.md](LICENSE.md). + +## CLI source + +`src/stac/cli.py` defines the console interface. Processing logic lives in `stac_impl.py`. Update the CLI and CWL bindings together when changing arguments. diff --git a/water-bodies/command-line-tools/stac/app.py b/water-bodies/command-line-tools/stac/app.py index d457174..cfe24d9 100644 --- a/water-bodies/command-line-tools/stac/app.py +++ b/water-bodies/command-line-tools/stac/app.py @@ -1,66 +1,5 @@ -"""Creates a STAC catalog with the detected water bodies""" "" -import os -import shutil -import click -import pystac -import rio_stac -from loguru import logger - - -@click.command( - short_help="Creates a STAC catalog", - help="Creates a STAC catalog with the water bodies", -) -@click.option( - "--input-item", - "item_urls", - help="STAC Item URL", - required=True, - multiple=True, -) -@click.option( - "--water-body", - "water_bodies", - help="Water body geotiff", - required=True, - multiple=True, -) -def to_stac(item_urls, water_bodies): - """Creates a STAC catalog with the detected water bodies""" - - logger.info(f"Creating a STAC Catalog for {' '.join(water_bodies)}") - cat = pystac.Catalog(id="catalog", description="water-bodies") - - for index, item_url in enumerate(item_urls): - if os.path.isdir(item_url): - catalog = pystac.read_file(os.path.join(item_url, "catalog.json")) - item = next(catalog.get_items()) - else: - item = pystac.read_file(item_url) - - water_body = water_bodies[index] - - os.makedirs(item.id, exist_ok=True) - shutil.copy(water_body, item.id) - - out_item = rio_stac.stac.create_stac_item( - source=water_body, - input_datetime=item.datetime, - id=item.id, - asset_roles=["data", "visual"], - asset_href=os.path.basename(water_body), - asset_name="data", - with_proj=True, - with_raster=True, - ) - - cat.add_items([out_item]) - - cat.normalize_and_save( - root_href="./", catalog_type=pystac.CatalogType.SELF_CONTAINED - ) - logger.info("Done!") - +"""Compatibility launcher; install the package and prefer `stac`.""" +from stac.cli import stac if __name__ == "__main__": - to_stac() + stac() diff --git a/water-bodies/command-line-tools/stac/pyproject.toml b/water-bodies/command-line-tools/stac/pyproject.toml new file mode 100644 index 0000000..b6a038a --- /dev/null +++ b/water-bodies/command-line-tools/stac/pyproject.toml @@ -0,0 +1,145 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stac" +dynamic = ["version"] +description = "Create a STAC catalog for water body masks" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "click==8.5.0", + "pystac==1.15.2", + "rio-stac==0.12.0", + "rasterio==1.5.2", + "loguru==0.7.3", +] + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/stac/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/stac/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + +[project.scripts] +stac = "stac.cli:stac" + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/stac tests}" + +[tool.coverage.run] +source_pkgs = ["stac", "tests"] +branch = true +parallel = true +omit = [ + "src/stac/__about__.py", +] + +[tool.coverage.paths] +stac = ["src/stac", "*/stac/src/stac"] +tests = ["tests", "*/stac/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=stac --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff] +# Generated by cwl2click; regenerate rather than hand-edit this file. +extend-exclude = ["src/stac/cli.py"] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stac/src/stac/__about__.py b/water-bodies/command-line-tools/stac/src/stac/__about__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/water-bodies/command-line-tools/stac/src/stac/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/stac/src/stac/__init__.py b/water-bodies/command-line-tools/stac/src/stac/__init__.py new file mode 100644 index 0000000..1dac57a --- /dev/null +++ b/water-bodies/command-line-tools/stac/src/stac/__init__.py @@ -0,0 +1 @@ +"""Create a STAC catalog for water body masks.""" diff --git a/water-bodies/command-line-tools/stac/src/stac/cli.py b/water-bodies/command-line-tools/stac/src/stac/cli.py new file mode 100644 index 0000000..3cc7048 --- /dev/null +++ b/water-bodies/command-line-tools/stac/src/stac/cli.py @@ -0,0 +1,54 @@ +# File generated by cwl2click v0.10.0 +# timestamp: 2026-09-23T15:58:42.074 + +__all__ = [] + +from stac.stac_impl import execute as stac_command + +from pathlib import Path + +import click + + + + + + + +# None +__all__.append("stac") +# NOTE +# Do not forget to add the section below in your `pyproject.toml` file +# +# [project.scripts] +# stac = "stac.cli:stac" + + +stac = click.Command( + name="stac", + callback=stac_command, + help="""Generate a STAC catalog for the water bodies using the source STAC items and the binary water body masks.""", + short_help="""STAC Generation""", + params=[ + click.Option( + ["--item"], + "item", + type=click.STRING, + multiple=True, + required=True, + is_flag=False, + help="""Source STAC item URLs in the same order as the water body masks.""", + ), + click.Option( + ["--rasters"], + "rasters", + type=click.Path(path_type=Path, exists=True, readable=True, resolve_path=True, file_okay=True, dir_okay=False), + multiple=True, + required=True, + is_flag=False, + help="""Binary water body masks in the same order as the source STAC items.""", + ), + ] +) + + diff --git a/water-bodies/command-line-tools/stac/src/stac/stac_impl.py b/water-bodies/command-line-tools/stac/src/stac/stac_impl.py new file mode 100644 index 0000000..7d33399 --- /dev/null +++ b/water-bodies/command-line-tools/stac/src/stac/stac_impl.py @@ -0,0 +1,76 @@ +"""Package water masks into a self-contained STAC catalog.""" + +import shutil +from pathlib import Path + +import click +import pystac +import rasterio +from loguru import logger +from rio_stac.stac import create_stac_item + + +def _read_item(location: str) -> pystac.Item: + path = Path(location) + obj = pystac.read_file(str(path / "catalog.json") if path.is_dir() else location) + if isinstance(obj, pystac.Catalog): + obj = next(obj.get_items(recursive=True), None) + if not isinstance(obj, pystac.Item): + raise ValueError("Input must contain a STAC item") + return obj + + +def _add_item(catalog: pystac.Catalog, item: pystac.Item, raster: str | Path) -> None: + directory = Path(item.id) + if directory.name != item.id or item.id in {"", ".", ".."}: + raise ValueError("STAC item id must be a single directory name") + directory.mkdir(exist_ok=True) + destination = directory / Path(raster).name + if Path(raster).resolve() != destination.resolve(): + shutil.copy(raster, destination) + logger.info("Adding water mask for item {}", item.id) + output = create_stac_item( + source=str(destination), + input_datetime=item.datetime, + id=item.id, + asset_roles=["data", "visual"], + asset_href=destination.name, + asset_name="data", + with_proj=True, + with_raster=True, + ) + catalog.add_item(output) + + +def _create_catalog( + item_urls: tuple[str, ...], water_bodies: tuple[str | Path, ...] +) -> None: + if not item_urls or len(item_urls) != len(water_bodies): + raise ValueError( + "Provide the same nonzero number of STAC items and water body rasters" + ) + items = [_read_item(url) for url in item_urls] + if len({item.id for item in items}) != len(items): + raise ValueError("STAC item ids must be unique") + catalog = pystac.Catalog(id="catalog", description="water-bodies") + for item, raster in zip(items, water_bodies, strict=True): + _add_item(catalog, item, raster) + catalog.normalize_and_save( + root_href="./", catalog_type=pystac.CatalogType.SELF_CONTAINED + ) + + +def execute(*, item: tuple[str, ...], rasters: tuple[Path, ...]) -> None: + """Create catalog.json and one item directory per water mask.""" + try: + logger.info("Creating catalog for {} water masks", len(rasters)) + _create_catalog(item, rasters) + logger.success("Wrote catalog.json") + except ( + ValueError, + OSError, + pystac.STACError, + rasterio.errors.RasterioError, + ) as exc: + logger.error("STAC catalog creation failed: {}", exc) + raise click.ClickException(str(exc)) from exc diff --git a/water-bodies/command-line-tools/stac/tests/__init__.py b/water-bodies/command-line-tools/stac/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/water-bodies/command-line-tools/stac/tests/test_stac.py b/water-bodies/command-line-tools/stac/tests/test_stac.py new file mode 100644 index 0000000..95608cb --- /dev/null +++ b/water-bodies/command-line-tools/stac/tests/test_stac.py @@ -0,0 +1,76 @@ +"""Exercise raster processing with local fixtures.""" + +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pystac +import pytest +import rasterio +from click.testing import CliRunner +from rasterio.transform import from_origin + +from stac.cli import stac + + +@pytest.fixture +def raster_factory(tmp_path): + def write(name, data): + path = tmp_path / name + data = np.asarray(data, dtype="float32") + with rasterio.open( + path, + "w", + driver="GTiff", + width=data.shape[1], + height=data.shape[0], + count=1, + dtype="float32", + crs="EPSG:4326", + transform=from_origin(0, 2, 1, 1), + ) as dst: + dst.write(data, 1) + return str(path) + + return write + + +@pytest.mark.parametrize("staged", [False, True]) +def test_catalog(raster_factory, tmp_path, monkeypatch, staged): + raster = raster_factory("mask.tif", [[0, 1], [1, 0]]) + source = pystac.Item( + "scene", + geometry=None, + bbox=None, + datetime=datetime(2026, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + source.set_self_href(str(tmp_path / "source.json")) + source.save_object() + location = source.get_self_href() + if staged: + catalog = pystac.Catalog("source", "Source catalog") + catalog.add_item(source) + catalog.set_self_href(str(tmp_path / "catalog.json")) + catalog.save_object() + location = str(tmp_path) + work = tmp_path / "output" + work.mkdir() + monkeypatch.chdir(work) + result = CliRunner().invoke(stac, ["--item", location, "--rasters", raster]) + assert result.exit_code == 0, result.output + catalog = pystac.Catalog.from_file("catalog.json") + item = next(catalog.get_items()) + assert item.id == "scene" + assert item.datetime == source.datetime + assert Path(item.assets["data"].get_absolute_href()).is_file() + assert Path(item.assets["data"].href) == Path("mask.tif") + + +def test_mismatched_counts(raster_factory): + raster = raster_factory("mask.tif", [[0, 1]]) + result = CliRunner().invoke( + stac, ["--item", "a", "--item", "b", "--rasters", raster] + ) + assert result.exit_code == 1 + assert "same nonzero number" in result.output From 45470be1dd801de1184c0007fd2a67ebcdef58e8 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:34:34 +0200 Subject: [PATCH 10/23] ci(processing): test packages and pack standalone CWL releases Test the synthetic processing chain, build wheels and images, update mandatory Docker requirements with release digests, and preserve metadata while embedding workflow schemas. --- .github/workflows/build.yaml | 16 +++++----- .github/workflows/processing-tests.yaml | 42 +++++++++++++++++++++++++ docs/release/ci.md | 9 +++++- scripts/pack-cwl-release.py | 19 +++++++++++ 4 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/processing-tests.yaml create mode 100644 scripts/pack-cwl-release.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 514b972..8963005 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,7 +10,8 @@ on: - 'water-bodies/**' - .github/** - docs/** - - cwl-workflow/*.cwl + - cwl-workflow/** + - scripts/pack-cwl-release.py - codemeta.json jobs: validate: @@ -109,7 +110,7 @@ jobs: shatag=$( docker inspect ${tag} | yq -r '.[0]["RepoDigests"][0]' ) for cwl in $(ls cwl-workflow/*.cwl) do - s="${step}" t="${shatag}" yq -i eval '(.$graph[] | select (.id == env(s)) ).hints.DockerRequirement.dockerPull = env(t)' $cwl + s="${step}" t="${shatag}" yq -i eval '(.$graph[] | select (.id == env(s)) ).requirements.DockerRequirement.dockerPull = env(t)' $cwl done done - run: | @@ -117,18 +118,19 @@ jobs: do r=$( cat codemeta.json | jq -r ".codeRepository" ) yq -i eval '."s:codeRepository" = {"URL" : env(r)}' $cwl v="${{needs.version.outputs.app-version}}" yq -i eval '."s:softwareVersion" = env(v)' $cwl - n=$(cat codemeta.json | jq -r '(.author[0].givenName + " " + .author[0].familyName)') \ - e=$(cat codemeta.json | jq -r '.author[0].email') \ - a=$(cat codemeta.json | jq -r '.author[0].affiliation["name"]') \ - yq eval -i '."s:author" += [{"class": "s:Person", "s.name": env(n), "s.email": env(e), "s.affiliation": env(a)}]' $cwl + done + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install cwltool - name: Cleanup downloads folder run: rm -rf downloads/* - run: | mkdir downloads for cwl in "app-water-bodies-cloud-native" "app-water-body-cloud-native" "app-water-body" do - cp cwl-workflow/${cwl}.cwl downloads/${cwl}.${{needs.version.outputs.app-version}}.cwl + python scripts/pack-cwl-release.py cwl-workflow/${cwl}.cwl downloads/${cwl}.${{needs.version.outputs.app-version}}.cwl done - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/processing-tests.yaml b/.github/workflows/processing-tests.yaml new file mode 100644 index 0000000..99c1ea8 --- /dev/null +++ b/.github/workflows/processing-tests.yaml @@ -0,0 +1,42 @@ +name: Processing packages +on: + pull_request: + paths: + - 'water-bodies/command-line-tools/**' + - 'cwl-cli/**' + - 'cwl-workflow/**' + - 'tests/**' + - '.github/workflows/processing-tests.yaml' + push: + branches: [main, master] + paths: + - 'water-bodies/command-line-tools/**' + - 'cwl-cli/**' + - 'cwl-workflow/**' + - 'tests/**' + - '.github/workflows/processing-tests.yaml' +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install packages and test tools + run: | + python -m pip install pytest build cwltool \ + ./water-bodies/command-line-tools/crop \ + ./water-bodies/command-line-tools/norm_diff \ + ./water-bodies/command-line-tools/otsu \ + ./water-bodies/command-line-tools/stac + - name: Test commands and processing chain + run: pytest --import-mode=importlib tests water-bodies/command-line-tools/{crop,norm_diff,otsu,stac}/tests + - name: Validate CWL and build wheels and images + run: | + while IFS= read -r cwl; do cwltool --validate "$cwl"; done < <(git ls-files '*.cwl') + for package in crop norm_diff otsu stac; do + python -m build --wheel --outdir /tmp/wheels "water-bodies/command-line-tools/$package" + docker build -t "processing-$package:test" "water-bodies/command-line-tools/$package" + docker run --rm "processing-$package:test" "$package" --help + done diff --git a/docs/release/ci.md b/docs/release/ci.md index 2c5963f..08342f5 100644 --- a/docs/release/ci.md +++ b/docs/release/ci.md @@ -51,4 +51,11 @@ Below an example of a GitHub CI configuration implementing the scenario: --8<-- .github/workflows/build.yaml --8<-- -``` \ No newline at end of file +``` + +The processing test workflow installs and tests the four Python packages, runs +a synthetic processing chain, validates the CWL, and builds wheels and images. +Release image digests are written to mandatory `requirements.DockerRequirement` +entries. `scripts/pack-cwl-release.py` selects the workflow explicitly, embeds +the schema imports, preserves package metadata, and validates the downloaded +artifact as a standalone CWL file. diff --git a/scripts/pack-cwl-release.py b/scripts/pack-cwl-release.py new file mode 100644 index 0000000..b5aed3d --- /dev/null +++ b/scripts/pack-cwl-release.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Pack a workflow and preserve its package metadata in the release artifact.""" +import json +import subprocess +import sys +from pathlib import Path + +from ruamel.yaml import YAML + +source, destination = map(Path, sys.argv[1:]) +packed = json.loads(subprocess.check_output([ + 'cwltool', '--pack', f'{source}#water-bodies'], text=True)) +metadata = YAML(typ='safe').load(source.read_text()) +for key, value in metadata.items(): + if key not in {'$graph', 'cwlVersion'}: + packed[key] = value +# Packing renames the selected entrypoint to #main and embeds local schema imports. +destination.write_text(json.dumps(packed, indent=2) + '\n') +subprocess.run(['cwltool', '--validate', str(destination)], check=True) From 54230f8a0721af4c34ccabc2652875936b68731c Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:34:37 +0200 Subject: [PATCH 11/23] docs(labs): align examples with packaged processing commands Install processing packages in lab environments, use the new CLI options and host-user container mapping, explain polygon masking, and refresh CWL references and diagrams. --- docs/app/crop/app.md | 68 +++++-------------- docs/app/normalized-difference/app.md | 41 ++++------- docs/app/otsu/app.md | 42 ++++-------- docs/app/stac/app.md | 47 ++++--------- docs/app/water-bodies-detection.md | 27 ++++++++ docs/containers/crop.md | 7 +- docs/cwl-cli/crop.md | 6 +- docs/python-env/crop.md | 2 +- docs/python-env/norm-diff.md | 2 +- docs/python-env/otsu.md | 2 +- docs/python-env/stac.md | 2 +- .../detect_water_body.md | 10 +-- .../ogc_processes_inputs.svg | 2 +- .../water-bodies.md | 14 ++-- .../water-bodies/ogc_processes_inputs.svg | 2 +- .../water-bodies.md | 20 +++--- .../water-bodies/ogc_processes_inputs.svg | 2 +- .../app-water-body/water-bodies.md | 20 +++--- .../water-bodies/ogc_processes_inputs.svg | 2 +- docs/reference/cwl/index.md | 3 +- practice-labs/1-Application_Steps/crop.ipynb | 19 +++--- .../normalized-difference.ipynb | 12 ++-- practice-labs/1-Application_Steps/otsu.ipynb | 10 ++- practice-labs/1-Application_Steps/stac.ipynb | 12 ++-- practice-labs/2-Containers/crop.ipynb | 20 ++---- .../2-Containers/normalized-difference.ipynb | 13 ++-- practice-labs/2-Containers/otsu.ipynb | 11 ++- practice-labs/2-Containers/stac.ipynb | 13 ++-- .../3-CWL-CommandLineTools/crop.ipynb | 2 +- .../4-CWL-Workflows/1-cloud-native.ipynb | 2 +- .../4-CWL-Workflows/2.2-staged.ipynb | 2 +- .../3-cloud-native-scatter.ipynb | 2 +- scripts/crop_env.sh | 2 +- scripts/crop_green.sh | 5 +- scripts/crop_nir.sh | 5 +- scripts/norm_diff.sh | 7 +- scripts/norm_diff_env.sh | 2 +- scripts/otsu.sh | 5 +- scripts/otsu_env.sh | 2 +- scripts/podman-crop-green.sh | 9 +-- scripts/podman-crop-nir.sh | 9 +-- scripts/podman-norm-diff.sh | 11 ++- scripts/podman-otsu.sh | 9 +-- scripts/podman-stac.sh | 11 ++- scripts/stac.sh | 7 +- scripts/stac_env.sh | 2 +- 46 files changed, 213 insertions(+), 312 deletions(-) diff --git a/docs/app/crop/app.md b/docs/app/crop/app.md index 931d0cf..ac275b0 100644 --- a/docs/app/crop/app.md +++ b/docs/app/crop/app.md @@ -26,63 +26,31 @@ end ### Code -The `crop.py` script is a command-line tool that takes as input +Crops a selected STAC asset to a GeoJSON Polygon or bounding box. The asset must have a data role and matching eo:bands or bands common_name. Relative asset paths are resolved against the item; staged catalogs select their first item. Planetary Computer URLs are signed when needed. The geometry is transformed into the raster CRS and rasterio masks pixels outside it. Remote reads use bounded retries and timeouts. The output crop_.tif retains the source datatype and nodata value and uses LZW-compressed COG format. -* a SpatioTemporal Asset Catalog (STAC) Item -* a bounding box area of interest (AOI), an EPSG code -* a common band name as input +The installed console interface is: -and then crops the specified band from the asset associated with the common band name to the specified AOI. +```text +Usage: crop [OPTIONS] -It uses various Python libraries like `pystac`, `rasterio`, `pyproj`, `shapely`, and `loguru`. - -Here is an overview of the script's functionality: - -* It defines a function `aoi2box` to convert an AOI expressed as a bounding box string into a list of floats. - -* It defines a function `get_asset` to retrieve the asset of a STAC Item that is defined with a common band name. It iterates through the assets and checks if a band has the specified common name. - -* It defines a command-line interface using `click`, with options for providing the input STAC Item URL, AOI, EPSG code, and common band name. - -* The `crop` function is the main entry point. It reads the STAC Item specified by the input URL and retrieves the asset associated with the common band name. It then crops the asset to the specified AOI using the rasterio library. - -* It transforms the bounding box coordinates to match the EPSG code provided. - -* It performs the cropping using the `rasterio.mask.mask` function. - -* It writes the cropped image to a GeoTIFF file with a filename like "crop_bandname.tif." - -The script is executable as a command-line tool as its usage is: - -``` -Usage: app.py [OPTIONS] - - Crops a STAC Item asset defined with its common band name + Crop each requested spectral band to the area of interest. Options: - --input-item TEXT STAC Item URL or staged STAC catalog [required] - --aoi TEXT Area of interest expressed as a bounding box [required] - --epsg TEXT EPSG code [required] - --band TEXT Common band name [required] - --help Show this message and exit. + --input-item TEXT URL of the source STAC item. [default: (item); + required] + --aoi TEXT GeoJSON polygon defining the area of interest. + [default: (aoi); required] + --epsg [4326] Coordinate reference system of the area of + interest. [default: (epsg); required] + --band [green|nir|nir08] Common name of the spectral band to crop. + [default: (band); required] + --help Show this message and exit. ``` -To use this script, you would typically run it from the command line, providing the necessary input options such as the STAC Item URL, AOI, EPSG code, and common band name: +The processing implementation is: -``` -python app.py \ - --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --aoi "-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ - --band "green" -``` - -It will then crop the specified band from the STAC asset and save it as a GeoTIFF file. - -The Python code is provided here: - -```python linenums="1" title="water-bodies/command-line-tools/crop/app.py" +```python linenums="1" title="crop_impl.py" --8<-- -water-bodies/command-line-tools/crop/app.py +water-bodies/command-line-tools/crop/src/crop/crop_impl.py --8<-- -``` \ No newline at end of file +``` diff --git a/docs/app/normalized-difference/app.md b/docs/app/normalized-difference/app.md index 730ce5b..7db0a64 100644 --- a/docs/app/normalized-difference/app.md +++ b/docs/app/normalized-difference/app.md @@ -25,41 +25,26 @@ end ### Code -The `norm_diff` script is a command-line tool for performing a normalized difference between two raster images. +Computes (first - second) / (first + second) from exactly two rasters supplied with repeated --rasters options. Dimensions, transforms, and coordinate systems must match. The output norm_diff.tif is a float32, LZW-compressed COG; zero denominators produce nonfinite values handled by the threshold step. -It uses the `click`, `rasterio`, and `numpy` libraries to perform the calculation and save the result as a GeoTIFF file. +The installed console interface is: -Here's an overview of what the script does: +```text +Usage: norm_diff [OPTIONS] -* It defines a command-line interface using the `click` library, with two arguments for providing the file paths of the two raster images that you want to calculate the normalized difference for. - -* The `normalized_difference` function is the main entry point. It opens the two input raster files specified as arguments. - -* It reads the data from the first raster (specified by `rasters[0]`) using `rasterio`, and it also copies the metadata (e.g., `projection`, `geotransform`) of this raster to be used for the output. - -* It then opens the second raster (specified by `rasters[1]`) and reads its data. - -* It updates the data type in the metadata to `"float32"` because the normalized difference result will be a floating-point image. - -* It creates an output raster named "norm_diff.tif" using `rasterio`. This output raster will have the same metadata as the first input raster, but it will be of data type `float32`. - -* It calculates the normalized difference between the two input arrays `(array1 - array2) / (array1 + array2)` and writes it to the output raster using `dst_dataset.write()`. - -The script is executable as a command-line tool as its usage is: - -``` -Usage: app.py [OPTIONS] RASTERS... - - Performs a normalized difference + Calculate the normalized difference water index (NDWI) from the green and + near-infrared spectral band rasters. Options: - --help Show this message and exit. + --rasters FILE Ordered green and near-infrared GeoTIFFs used to calculate + NDWI. [default: (rasters); required] + --help Show this message and exit. ``` -The Python code is provided here: +The processing implementation is: -```python linenums="1" title="water-bodies/command-line-tools/norm_diff/app.py" +```python linenums="1" title="norm_diff_impl.py" --8<-- -water-bodies/command-line-tools/norm_diff/app.py +water-bodies/command-line-tools/norm_diff/src/norm_diff/norm_diff_impl.py --8<-- -``` \ No newline at end of file +``` diff --git a/docs/app/otsu/app.md b/docs/app/otsu/app.md index f30d943..79c60c5 100644 --- a/docs/app/otsu/app.md +++ b/docs/app/otsu/app.md @@ -25,42 +25,26 @@ end ### Code -The `otsu` Python script is a command-line tool for applying the Otsu threshold to a single input raster image. +Reads the raster supplied with --raster and computes the Otsu threshold from finite pixel values. Pixels above the threshold become 1; other pixels, including nonfinite values, become 0. An input with no finite values fails. The output otsu.tif retains the spatial metadata and uses uint8, LZW-compressed COG format without a nodata value. -It uses the `click`, `rasterio`, `numpy`, `skimage.filters`, and `loguru` libraries. +The installed console interface is: -Here's an overview of what the script does: +```text +Usage: otsu [OPTIONS] -* It defines a command-line interface using the `click` library, with a single argument for providing the file path of the input raster image on which you want to apply the Otsu threshold. - -* The `otsu` function is the main entry point. It opens the input raster file specified as the argument. - -* It reads the data from the input raster using `rasterio` and also copies the metadata (e.g., `projection`, `geotransform`) of this raster to be used for the output. - -* It applies the Otsu threshold to the input array by calling the `threshold` function. The `threshold_otsu` function from `skimage.filters` is used to calculate the Otsu threshold. The thresholding process marks pixels as True or False based on whether they are greater than the calculated threshold. - -* It creates an output raster named "otsu.tif" using `rasterio`. This output raster will have the same metadata as the input raster. - -* It writes the binary image to the output raster using `dst_dataset.write()`. - -The result, a binary image where pixel values are either True or False based on the thresholding, will be saved as "otsu.tif" in the same directory where the script is executed. - - -The script is executable as a command-line tool as its usage is: - -``` -Usage: app.py [OPTIONS] RASTER - - Applies the Otsu threshold + Apply Otsu thresholding to the NDWI raster to generate a binary water body + mask. Options: - --help Show this message and exit. + --raster FILE Normalized difference water index raster to threshold. + [default: (raster); required] + --help Show this message and exit. ``` -The Python code is provided here: +The processing implementation is: -```python linenums="1" title="water-bodies/command-line-tools/otsu/app.py" +```python linenums="1" title="otsu_impl.py" --8<-- -water-bodies/command-line-tools/otsu/app.py +water-bodies/command-line-tools/otsu/src/otsu/otsu_impl.py --8<-- -``` \ No newline at end of file +``` diff --git a/docs/app/stac/app.md b/docs/app/stac/app.md index a9a1d07..14b7184 100644 --- a/docs/app/stac/app.md +++ b/docs/app/stac/app.md @@ -25,45 +25,28 @@ end ### Code -The `stac` script is a command-line tool for creating a SpatioTemporal Asset Catalog (STAC) catalog containing detected water bodies. +Pairs source STAC items with masks using repeated --item and --rasters options in corresponding order. It accepts item URLs, local files, and staged catalog directories. Item counts must match raster counts, and IDs must be unique directory names. Each raster is copied into its item directory, with projection and raster metadata derived by rio-stac. The output catalog.json and item directories form a self-contained STAC catalog with relative asset links. -It takes the STAC Item URLs and corresponding water body GeoTIFFs as input, creates STAC Items for each water body, and organizes them into a STAC Catalog. +The installed console interface is: -The script utilizes the `click`, `pystac`, `rio_stac`, and `loguru` libraries. +```text +Usage: stac [OPTIONS] -Here's an overview of what the script does: - -* It defines a command-line interface using the `click` library. The script expects multiple input STAC Item URLs and corresponding water body GeoTIFFs as arguments. - -* The `to_stac` function is the main entry point. It creates a new STAC catalog and iterates through the provided STAC Item URLs and water body GeoTIFFs. - -* For each input pair (STAC Item URL and GeoTIFF), it reads the STAC Item, creates a directory with the same name as the item's ID, and copies the water body GeoTIFF into that directory. - -* It uses the `rio_stac` library to create a STAC Item for the water body. This includes specifying the source GeoTIFF, input date/time, asset roles, asset href, and more. - -* The created STAC Items are added to the STAC catalog. - -* After processing all input pairs, it saves the STAC catalog to the root directory, specifying it as a self-contained catalog with the catalog type set to `pystac.CatalogType.SELF_CONTAINED`. - -The script will create a STAC catalog containing the detected water bodies and save it in the current directory. - -The script is executable as a command-line tool as its usage is: - -``` -Usage: app.py [OPTIONS] - - Creates a STAC catalog with the water bodies + Generate a STAC catalog for the water bodies using the source STAC items and + the binary water body masks. Options: - --input-item TEXT STAC Item URL [required] - --water-body TEXT Water body geotiff [required] - --help Show this message and exit. + --item TEXT Source STAC item URLs in the same order as the water body + masks. [default: (item); required] + --rasters FILE Binary water body masks in the same order as the source STAC + items. [default: (rasters); required] + --help Show this message and exit. ``` -The Python code is provided here: +The processing implementation is: -```python linenums="1" title="water-bodies/command-line-tools/stac/app.py" +```python linenums="1" title="stac_impl.py" --8<-- -water-bodies/command-line-tools/stac/app.py +water-bodies/command-line-tools/stac/src/stac/stac_impl.py --8<-- -``` \ No newline at end of file +``` diff --git a/docs/app/water-bodies-detection.md b/docs/app/water-bodies-detection.md index 37a8aaa..652ae79 100644 --- a/docs/app/water-bodies-detection.md +++ b/docs/app/water-bodies-detection.md @@ -106,3 +106,30 @@ And one Landsat-9 acquisition: Each `Command Line Tool` step such as `crop`, `Normalized difference`, `Otsu threshold` and `Create STAC` runs a simple Python script in a dedicated container. + + +## Processing packages + +The crop, normalized difference, Otsu, and STAC steps are Python packages with +`pyproject.toml`, a `src/` layout, Hatchling wheel builds, and local tests. +Python 3.12 or newer is required. Install a step with +`python -m pip install -e water-bodies/command-line-tools/crop` (substitute the +package directory for other steps). The practice notebooks create isolated +virtual environments and install the package instead of separate dependencies. + +The installed commands are `crop`, `norm_diff`, `otsu`, and `stac`. +Use repeated `--rasters` options for the two normalized difference inputs, +`otsu --raster norm_diff.tif`, and `stac --item source.json --rasters otsu.tif`. +Crop accepts Polygon JSON or a bounding box, with `--epsg 4326`; CWL passes the +full Polygon. Pixels outside the polygon are masked, and the source pixel type +is retained. Planetary Computer assets are signed when needed; remote reads +have bounded timeouts and retries. + +The Dockerfiles build wheels in a separate builder and install them in +`/app/venv` on the pinned Python 3.12 slim base. Images run as UID/GID 2000 +and expose console commands on PATH. The Podman examples map the host user +with `--userns=keep-id` so bind-mounted output folders remain writable. +Rebuild the four local images before running the updated CWL documents; +previous images do not contain the installed console commands. + +Stage-in and stage-out retain their existing implementation. diff --git a/docs/containers/crop.md b/docs/containers/crop.md index 692ac1d..b657cd3 100644 --- a/docs/containers/crop.md +++ b/docs/containers/crop.md @@ -46,15 +46,14 @@ Let's break down what this command does: `--mount=type=bind,source=/workspace/mastering-app-package/runs,target=/runs`: This option mounts a directory from the host system to the container. In this case, it mounts the `/workspace/mastering-app-package/runs` directory on the host to the /runs directory inside the container. * `--workdir=/runs`: Sets the working directory inside the container to `/runs`. * `--read-only=true`: Makes the file system inside the container read-only, meaning you can't write or modify files inside the container. -* `--user=1001:100`: Specifies the user and group IDs to be used within the container. +* `--user="$(id -u):$(id -g)"`: Specifies the user and group IDs to be used within the container. * `--rm`: This flag tells podman to remove the container after it has finished running. * `--env=HOME=/runs`: Sets the `HOME` environment variable inside the container to `/runs`. -* `--env=PYTHONPATH=/app`: Sets the `PYTHONPATH` environment variable inside the container to `/app`. * `localhost/crop:latest`: This is the name of the container image that you want to run. It's pulling the image from the local container registry with the name "crop" and the "latest" tag. -* `python -m app`: This is the command to run inside the container. It runs a Python module named "app." +* `crop`: Runs the installed console command. * `--aoi "-121.399,39.834,-120.74,40.472"`: This provides command-line arguments to the Python module. It specifies the area of interest (AOI) as a bounding box. * `--band green`: Specifies the band to be extracted from the Sentinel-2 acquisition. In this case, it's the "green" band. -* `--epsg "EPSG:4326"``: Specifies the EPSG code, which defines the coordinate system used for the `aoi` command-line argument. +* `--epsg "4326"`: Specifies the EPSG code, which defines the coordinate system used for the `aoi` command-line argument. * `--input-item ...`: Specifies the input STAC item URL. This particular URL points to a Sentinel-2 image hosted on AWS Earth Search. ### Steps diff --git a/docs/cwl-cli/crop.md b/docs/cwl-cli/crop.md index 77822d3..7674f60 100644 --- a/docs/cwl-cli/crop.md +++ b/docs/cwl-cli/crop.md @@ -24,14 +24,14 @@ Let's break down the key components of this CWL document: * `id: crop`: Provides a unique identifier for this tool, which can be used to reference it in workflows. * `requirements`: Specifies the requirements and dependencies of the tool. In this case, it defines the following: * `InlineJavascriptRequirement`: This requirement allows the use of inline JavaScript expressions in the tool. - * `EnvVarRequirement`: It sets environment variables. In this case, it sets the `PYTHONPATH` environment variable to "/app." + * `EnvVarRequirement`: It sets environment variables. In this case, it adds `/app/venv/bin` to `PATH` so the installed commands can be found. * `ResourceRequirement`: Specifies resource requirements for running the tool, including the maximum number of CPU cores and maximum RAM. * `DockerRequirement`: This requirement specifies the Docker container to be used. It indicates that the tool should be executed in a Docker container with the image `localhost/crop:latest`. -* `baseCommand`: Defines the base command to be executed in the container. In this case, it's running a Python module called "app" with the command `python -m app`. +* `baseCommand`: Defines the base command to be executed in the container. In this case, it runs the installed `crop` command. * `arguments`: This section is empty, meaning there are no additional command-line arguments specified here. The tool is expected to receive its arguments via the input parameters. * `inputs`: Describes the input parameters for the tool, including their types and how they are bound to command-line arguments. The tool expects the following inputs: * `item`: A URI record containing a `value` field with the STAC item URL; `valueFrom: $(self.value)` passes that URL, bound to the `--input-item` argument. - * `aoi`: A GeoJSON Polygon record with `coordinates` and `bbox`; `valueFrom: $(self.bbox.join(","))` converts its bounding box to the application argument, bound to the `--aoi` argument. + * `aoi`: A GeoJSON Polygon record with `coordinates` and `bbox`; `valueFrom: $(JSON.stringify(self))` serializes the full Polygon, bound to the `--aoi` argument. * `epsg`: A string representing the EPSG code for the coordinate system, bound to the `--epsg` argument. * `band`: A string representing the name of the band to be extracted, bound to the `--band` argument. * `outputs`: Specifies the tool's output. It defines an output parameter named `cropped`, which is of type `File`. The outputBinding section specifies that the tool is expected to produce one or more TIFF files (glob: '*.tif') as output. diff --git a/docs/python-env/crop.md b/docs/python-env/crop.md index 9331809..000add1 100644 --- a/docs/python-env/crop.md +++ b/docs/python-env/crop.md @@ -22,7 +22,7 @@ source /workspace/mastering-app-package/scripts/setup.sh ### Step 2 - Create the Python virtual environment -The required Python modules are installed using `pip`: +Install the Python package and its pinned dependencies in the virtual environment using `pip`: ```bash linenums="1" hl_lines="3" title="terminal" --8<-- diff --git a/docs/python-env/norm-diff.md b/docs/python-env/norm-diff.md index 85948ab..f6aeb01 100644 --- a/docs/python-env/norm-diff.md +++ b/docs/python-env/norm-diff.md @@ -22,7 +22,7 @@ source /workspace/mastering-app-package/scripts/setup.sh ### Step 2 - Create the Python virtual environment -The required Python modules are installed using `pip`: +Install the Python package and its pinned dependencies in the virtual environment using `pip`: ```bash linenums="1" hl_lines="3" title="terminal" --8<-- diff --git a/docs/python-env/otsu.md b/docs/python-env/otsu.md index 23e74cd..d965901 100644 --- a/docs/python-env/otsu.md +++ b/docs/python-env/otsu.md @@ -22,7 +22,7 @@ source /workspace/mastering-app-package/scripts/setup.sh ### Step 2 - Create the Python virtual environment -The required Python modules are installed using `pip`: +Install the Python package and its pinned dependencies in the virtual environment using `pip`: ```bash linenums="1" hl_lines="3" title="terminal" --8<-- diff --git a/docs/python-env/stac.md b/docs/python-env/stac.md index 9ed235d..b7b841f 100644 --- a/docs/python-env/stac.md +++ b/docs/python-env/stac.md @@ -22,7 +22,7 @@ source /workspace/mastering-app-package/scripts/setup.sh ### Step 2 - Create the Python virtual environment -The required Python modules are installed using `pip`: +Install the Python package and its pinned dependencies in the virtual environment using `pip`: ```bash linenums="1" hl_lines="3" title="terminal" --8<-- diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md index 3bd098b..fce9ef5 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:49:46.289 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T20:29:45.005 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -58,7 +58,7 @@ The are no Supported Operating Systems specified for this project. | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. | | `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| Coordinate reference system | EPSG code | | `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| NDWI bands | bands used for the NDWI | | `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item | STAC item | @@ -153,7 +153,7 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram ### Execution usage example: ``` -python -m app \ +crop \ --input-item \ --aoi \ --epsg \ @@ -181,7 +181,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +norm_diff \ --rasters ``` @@ -206,6 +206,6 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +otsu \ --raster ``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg index acee659..1416942 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleArea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleCoordinate reference systemdescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleNDWI bandsdescriptionbands used for the NDWItypearrayitems   typestringenum   greennirnir08   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC itemdescriptionSTAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleArea of interestdescriptionGeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleCoordinate reference systemdescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleNDWI bandsdescriptionbands used for the NDWItypearrayitems   typestringenum   greennirnir08   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC itemdescriptionSTAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md index a78c4b7..4f84b64 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 CO > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:49:46.219 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T20:29:45.026 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -59,7 +59,7 @@ The are no Supported Operating Systems specified for this project. | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. | | `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | | `stac_items` | `array` of [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Sentinel-2 STAC items | list of Sentinel-2 COG STAC items | | `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | @@ -150,7 +150,7 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| Area of interest | GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. | | `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| Coordinate reference system | EPSG code | | `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| NDWI bands | bands used for the NDWI | | `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item | STAC item | @@ -245,7 +245,7 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram ### Execution usage example: ``` -python -m app \ +crop \ --input-item \ --aoi \ --epsg \ @@ -273,7 +273,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +norm_diff \ --rasters ``` @@ -298,7 +298,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +otsu \ --raster ``` @@ -326,7 +326,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +stac \ --item \ --rasters ``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg index e395bc2..11dc6c6 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   stac_items   bands   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSentinel-2 STAC itemsdescriptionlist of Sentinel-2 COG STAC itemstypearrayitems   typestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[] \ No newline at end of file +aoi   epsg   stac_items   bands   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSentinel-2 STAC itemsdescriptionlist of Sentinel-2 COG STAC itemstypearrayitems   typestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[] \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md index 960c1b0..2cd8239 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to a single Sent > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:49:51.045 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T20:29:47.518 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -58,7 +58,7 @@ The are no Supported Operating Systems specified for this project. | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. | | `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | | `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | | `item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| STAC item reference | Reference to a STAC item | @@ -154,7 +154,7 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram ### Execution usage example: ``` -python -m app \ +crop \ --input-item \ --aoi \ --epsg \ @@ -182,7 +182,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +norm_diff \ --rasters ``` @@ -207,7 +207,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +otsu \ --raster ``` @@ -227,13 +227,13 @@ python -m app \ | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| -| `rasters` | `--water-body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | +| `item` | `--item` | [URI](https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI):
  • `value`: [string](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| +| `rasters` | `--rasters` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | ### Execution usage example: ``` -python -m app \ ---input-item \ ---water-body +stac \ +--item \ +--rasters ``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg index be4fd3d..ba8218d 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemtypestringformaturi   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI \ No newline at end of file diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md index 08f0555..947cee8 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md @@ -4,7 +4,7 @@ Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T18:49:54.541 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T20:29:49.248 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team @@ -58,7 +58,7 @@ The are no Supported Operating Systems specified for this project. | Id | Type | Label | Doc | |----|------|-------|-----| -| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system. | +| `aoi` | [Polygon](https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygon):
  • `type`: [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
    • `Polygon`
  • `coordinates`: `array` of `array` of `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
  • `bbox`: `array` of [double](https://www.commonwl.org/v1.2/Workflow.html#CWLType)
| area of interest | GeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked. | | `epsg` | [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `4326`
| EPSG code | EPSG code | | `bands` | `array` of [enum](https://www.commonwl.org/v1.2/Workflow.html#CommandInputEnumSchema):
  • `green`
  • `nir`
  • `nir08`
| bands used for the NDWI | bands used for the NDWI | | `item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | STAC item reference | Reference to a STAC item | @@ -154,7 +154,7 @@ Learn more about the [State diagram](https://en.wikipedia.org/wiki/State_diagram ### Execution usage example: ``` -python -m app \ +crop \ --input-item \ --aoi \ --epsg \ @@ -182,7 +182,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +norm_diff \ --rasters ``` @@ -207,7 +207,7 @@ python -m app \ ### Execution usage example: ``` -python -m app \ +otsu \ --raster ``` @@ -227,13 +227,13 @@ python -m app \ | Id | Option | Type | |----|------|-------| -| `item` | `--input-item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | -| `rasters` | `--water-body` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | +| `item` | `--item` | [Directory](https://www.commonwl.org/v1.2/Workflow.html#Directory) | +| `rasters` | `--rasters` | [File](https://www.commonwl.org/v1.2/Workflow.html#File) | ### Execution usage example: ``` -python -m app \ ---input-item \ ---water-body +stac \ +--item \ +--rasters ``` diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg index f65f233..35ae1f8 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies/ogc_processes_inputs.svg @@ -1 +1 @@ -aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon whose bbox defines the rectangular area to crop, in the specified coordinate reference system.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemoneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file +aoi   epsg   bands   item   schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlearea of interestdescriptionGeoJSON Polygon defining the area to crop; raster pixels outside the polygon are masked.typeobjectproperties   required   type   coordinates   bbox   typestringenum   Polygontypearrayitems   typearrayitems   typearrayitems   typenumberformatdoubletypearrayitems   typenumberformatdoubletypecoordinatesbbox   titlecwl:typevaluehttps://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml#Polygonschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleEPSG codedescriptionEPSG codetypestringenum   default43264326   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#EPSGCodeschema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitlebands used for the NDWIdescriptionbands used for the NDWItypearrayitems   default   typestringenum   greennirnir08greennir   titlecwl:typevaluefile:///data/work/github-com/eoap/mastering-app-package/cwl-workflow/eoap-types.yaml#SpectralBand[]schema   metadata   minOccurs1maxOccurs1valuePassingbyValuetitleSTAC item referencedescriptionReference to a STAC itemoneOf               typestringformaturi$refhttps://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json$refhttps://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json$refhttps://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml   titlecwl:typevalueDirectory \ No newline at end of file diff --git a/docs/reference/cwl/index.md b/docs/reference/cwl/index.md index 0fd6e7b..565f456 100644 --- a/docs/reference/cwl/index.md +++ b/docs/reference/cwl/index.md @@ -95,8 +95,7 @@ verified by static validation. For the multi-acquisition workflow, supply `stac_items` as a list of records with a `value` field instead of `item`. The imported Polygon schema requires both `coordinates` and `bbox`. The crop -binding passes `bbox` as comma-separated coordinates to the existing application; -cropping remains rectangular. URI records are unwrapped through `self.value`. +binding serializes the full Polygon as JSON; pixels outside the polygon are masked. URI records are unwrapped through `self.value`. These changes require updating jobs that previously supplied plain AOI and STAC strings. The imports reference the schemas repository's `main` branch. diff --git a/practice-labs/1-Application_Steps/crop.ipynb b/practice-labs/1-Application_Steps/crop.ipynb index 12d4385..c61b724 100644 --- a/practice-labs/1-Application_Steps/crop.ipynb +++ b/practice-labs/1-Application_Steps/crop.ipynb @@ -103,7 +103,7 @@ "outputs": [], "source": [ "\n", - "pip install --no-cache-dir rasterio==1.5.2 click==8.5.0 pystac==1.15.2 loguru==0.7.3 pyproj==3.8.0 shapely==2.1.2 " + "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/crop\"" ] }, { @@ -139,8 +139,7 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/crop/app.py --help" + "crop --help" ] }, { @@ -151,7 +150,7 @@ "\n", "- input-item: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n", "- aoi: -121.399,39.834,-120.74,40.472\n", - "- epsg: EPSG:4326\n", + "- epsg: 4326\n", "- band: green " ] }, @@ -166,11 +165,10 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/crop/app.py \\\n", + "crop \\\n", " --input-item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", " --aoi \"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\" \\\n", + " --epsg \"4326\" \\\n", " --band green \n" ] }, @@ -182,7 +180,7 @@ "\n", "- input-item: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\n", "- aoi: -121.399,39.834,-120.74,40.472\n", - "- epsg: EPSG:4326\n", + "- epsg: 4326\n", "- band: nir " ] }, @@ -196,11 +194,10 @@ }, "outputs": [], "source": [ - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/crop/app.py \\\n", + "crop \\\n", " --input-item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", " --aoi \"-121.399,39.834,-120.74,40.472\" \\\n", - " --epsg \"EPSG:4326\" \\\n", + " --epsg \"4326\" \\\n", " --band nir " ] }, diff --git a/practice-labs/1-Application_Steps/normalized-difference.ipynb b/practice-labs/1-Application_Steps/normalized-difference.ipynb index c14c241..2532b65 100644 --- a/practice-labs/1-Application_Steps/normalized-difference.ipynb +++ b/practice-labs/1-Application_Steps/normalized-difference.ipynb @@ -103,7 +103,7 @@ "outputs": [], "source": [ "\n", - "pip install --no-cache-dir rasterio==1.5.2 click==8.5.0 loguru==0.7.3" + "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/norm_diff\"" ] }, { @@ -139,8 +139,7 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/norm_diff/app.py --help" + "norm_diff --help" ] }, { @@ -154,10 +153,9 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/norm_diff/app.py \\\n", - " crop_green.tif \\\n", - " crop_nir.tif\n" + "norm_diff \\\n", + " --rasters crop_green.tif \\\n", + " --rasters crop_nir.tif\n" ] }, { diff --git a/practice-labs/1-Application_Steps/otsu.ipynb b/practice-labs/1-Application_Steps/otsu.ipynb index d6fdf05..393e0f2 100644 --- a/practice-labs/1-Application_Steps/otsu.ipynb +++ b/practice-labs/1-Application_Steps/otsu.ipynb @@ -103,7 +103,7 @@ "outputs": [], "source": [ "\n", - "pip install --no-cache-dir rasterio==1.5.2 scikit-image==0.26.0 click==8.5.0 loguru==0.7.3" + "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/otsu\"" ] }, { @@ -139,8 +139,7 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/otsu/app.py --help" + "otsu --help" ] }, { @@ -154,9 +153,8 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/otsu/app.py \\\n", - " norm_diff.tif\n" + "otsu \\\n", + " --raster norm_diff.tif\n" ] }, { diff --git a/practice-labs/1-Application_Steps/stac.ipynb b/practice-labs/1-Application_Steps/stac.ipynb index caffc4d..56f0b75 100644 --- a/practice-labs/1-Application_Steps/stac.ipynb +++ b/practice-labs/1-Application_Steps/stac.ipynb @@ -103,7 +103,7 @@ "outputs": [], "source": [ "\n", - "pip install --no-cache-dir rio_stac==0.12.0 loguru==0.7.3 click==8.5.0" + "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/stac\"" ] }, { @@ -139,8 +139,7 @@ "outputs": [], "source": [ "\n", - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/stac/app.py --help" + "stac --help" ] }, { @@ -153,10 +152,9 @@ }, "outputs": [], "source": [ - "python \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/stac/app.py \\\n", - " --input-item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --water-body otsu.tif\n" + "stac \\\n", + " --item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", + " --rasters otsu.tif\n" ] }, { diff --git a/practice-labs/2-Containers/crop.ipynb b/practice-labs/2-Containers/crop.ipynb index 493b639..0ac0e23 100644 --- a/practice-labs/2-Containers/crop.ipynb +++ b/practice-labs/2-Containers/crop.ipynb @@ -96,7 +96,7 @@ }, "outputs": [], "source": [ - "podman run --rm -it --env=PYTHONPATH=/app localhost/crop:latest python -m app --help" + "podman run --rm -itlocalhost/crop:latest crop --help" ] }, { @@ -124,20 +124,17 @@ " --mount=type=bind,source=/workspace/mastering-app-package/runs,target=/runs \\\n", " --workdir=/runs \\\n", " --read-only=true \\\n", - " --user=1001:100 \\\n", + " --user=\"$(id -u):$(id -g)\" \\\n", " --rm \\\n", " --env=HOME=/runs \\\n", - " --env=PYTHONPATH=/app \\\n", " localhost/crop:latest \\\n", - " python \\\n", - " -m \\\n", - " app \\\n", + " crop \\\n", " --aoi \\\n", " \"-121.399,39.834,-120.74,40.472\" \\\n", " --band \\\n", " green \\\n", " --epsg \\\n", - " \"EPSG:4326\" \\\n", + " \"4326\" \\\n", " --input-item \\\n", " https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" ] @@ -165,20 +162,17 @@ " --mount=type=bind,source=/workspace/mastering-app-package/runs,target=/runs \\\n", " --workdir=/runs \\\n", " --read-only=true \\\n", - " --user=1001:100 \\\n", + " --user=\"$(id -u):$(id -g)\" \\\n", " --rm \\\n", " --env=HOME=/runs \\\n", - " --env=PYTHONPATH=/app \\\n", " localhost/crop:latest \\\n", - " python \\\n", - " -m \\\n", - " app \\\n", + " crop \\\n", " --aoi \\\n", " \"-121.399,39.834,-120.74,40.472\" \\\n", " --band \\\n", " nir \\\n", " --epsg \\\n", - " \"EPSG:4326\" \\\n", + " \"4326\" \\\n", " --input-item \\\n", " https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" ] diff --git a/practice-labs/2-Containers/normalized-difference.ipynb b/practice-labs/2-Containers/normalized-difference.ipynb index 26520bc..0284d62 100644 --- a/practice-labs/2-Containers/normalized-difference.ipynb +++ b/practice-labs/2-Containers/normalized-difference.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -it --env=PYTHONPATH=/app localhost/norm-diff:latest python -m app --help" + "podman run --rm -itlocalhost/norm-diff:latest norm_diff --help" ] }, { @@ -126,16 +126,13 @@ " --mount=type=bind,source=/workspace/mastering-app-package/runs/crop_nir.tif,target=/inputs/crop_nir.tif,readonly \\\n", " --workdir=/runs \\\n", " --read-only=true \\\n", - " --user=1001:100 \\\n", + " --user=\"$(id -u):$(id -g)\" \\\n", " --rm \\\n", " --env=HOME=/runs \\\n", - " --env=PYTHONPATH=/app \\\n", " localhost/norm-diff:latest \\\n", - " python \\\n", - " -m \\\n", - " app \\\n", - " /inputs/crop_green.tif \\\n", - " /inputs/crop_nir.tif" + " norm_diff \\\n", + " --rasters /inputs/crop_green.tif \\\n", + " --rasters /inputs/crop_nir.tif" ] }, { diff --git a/practice-labs/2-Containers/otsu.ipynb b/practice-labs/2-Containers/otsu.ipynb index 2b4f86f..6678581 100644 --- a/practice-labs/2-Containers/otsu.ipynb +++ b/practice-labs/2-Containers/otsu.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -it --env=PYTHONPATH=/app localhost/otsu:latest python -m app --help" + "podman run --rm -itlocalhost/otsu:latest otsu --help" ] }, { @@ -125,15 +125,12 @@ " --mount=type=bind,source=/workspace/mastering-app-package/runs/norm_diff.tif,target=/inputs/norm_diff.tif,readonly \\\n", " --workdir=/runs \\\n", " --read-only=true \\\n", - " --user=1001:100 \\\n", + " --user=\"$(id -u):$(id -g)\" \\\n", " --rm \\\n", " --env=HOME=/runs \\\n", - " --env=PYTHONPATH=/app \\\n", " localhost/otsu:latest \\\n", - " python \\\n", - " -m \\\n", - " app \\\n", - " /inputs/norm_diff.tif" + " otsu \\\n", + " --raster /inputs/norm_diff.tif" ] }, { diff --git a/practice-labs/2-Containers/stac.ipynb b/practice-labs/2-Containers/stac.ipynb index a278aa7..437340a 100644 --- a/practice-labs/2-Containers/stac.ipynb +++ b/practice-labs/2-Containers/stac.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -it --env=PYTHONPATH=/app localhost/stac:latest python -m app --help" + "podman run --rm -itlocalhost/stac:latest stac --help" ] }, { @@ -125,17 +125,14 @@ " --mount=type=bind,source=/workspace/mastering-app-package/runs/otsu.tif,target=/inputs/otsu.tif,readonly \\\n", " --workdir=/runs \\\n", " --read-only=true \\\n", - " --user=1001:100 \\\n", + " --user=\"$(id -u):$(id -g)\" \\\n", " --rm \\\n", " --env=HOME=/runs \\\n", - " --env=PYTHONPATH=/app \\\n", " localhost/stac:latest \\\n", - " python \\\n", - " -m \\\n", - " app \\\n", - " --input-item \\\n", + " stac \\\n", + " --item \\\n", " https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A \\\n", - " --water-body \\\n", + " --rasters \\\n", " /inputs/otsu.tif" ] }, diff --git a/practice-labs/3-CWL-CommandLineTools/crop.ipynb b/practice-labs/3-CWL-CommandLineTools/crop.ipynb index 8e31a14..8cc78d0 100644 --- a/practice-labs/3-CWL-CommandLineTools/crop.ipynb +++ b/practice-labs/3-CWL-CommandLineTools/crop.ipynb @@ -16,7 +16,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding serializes the Polygon, and pixels outside it are masked. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb b/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb index 9f28257..4e06339 100644 --- a/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb +++ b/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb @@ -18,7 +18,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding serializes the Polygon, and pixels outside it are masked. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/2.2-staged.ipynb b/practice-labs/4-CWL-Workflows/2.2-staged.ipynb index dde8287..aa4e6ff 100644 --- a/practice-labs/4-CWL-Workflows/2.2-staged.ipynb +++ b/practice-labs/4-CWL-Workflows/2.2-staged.ipynb @@ -20,7 +20,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding serializes the Polygon, and pixels outside it are masked. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" ] }, { diff --git a/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb b/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb index a81eaa3..80e6590 100644 --- a/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb +++ b/practice-labs/4-CWL-Workflows/3-cloud-native-scatter.ipynb @@ -18,7 +18,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding uses `bbox`, so cropping remains rectangular. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" + "The local CWL documents use imported EOAP types. Supply a YAML job document: STAC references are URI records with a `value` field; AOIs are Polygon records with `type`, `coordinates`, and `bbox`. The crop binding serializes the Polygon, and pixels outside it are masked. Workflow EPSG values are `\"4326\"`, and band values are `green`, `nir`, or `nir08`. The standalone crop tool still accepts a string EPSG code. Staged acquisitions remain CWL Directory objects. Published release examples elsewhere retain their original input format.\n" ] }, { diff --git a/scripts/crop_env.sh b/scripts/crop_env.sh index 9ea926e..e88611b 100644 --- a/scripts/crop_env.sh +++ b/scripts/crop_env.sh @@ -1,3 +1,3 @@ python -m venv env_crop source env_crop/bin/activate -pip install --no-cache-dir rasterio click pystac loguru pyproj shapely +python -m pip install -e "${WORKSPACE}/water-bodies/command-line-tools/crop" diff --git a/scripts/crop_green.sh b/scripts/crop_green.sh index 2badc48..3c85b23 100644 --- a/scripts/crop_green.sh +++ b/scripts/crop_green.sh @@ -3,9 +3,8 @@ export RUNTIME=/workspace/mastering-app-package/runs mkdir -p ${RUNTIME} cd ${RUNTIME} -python \ - ${WORKSPACE}/water-bodies/command-line-tools/crop/app.py \ +crop \ --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ --aoi "-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ + --epsg "4326" \ --band green \ No newline at end of file diff --git a/scripts/crop_nir.sh b/scripts/crop_nir.sh index 0882f08..507e9d2 100644 --- a/scripts/crop_nir.sh +++ b/scripts/crop_nir.sh @@ -3,9 +3,8 @@ export RUNTIME=${WORKSPACE}/runs mkdir -p ${RUNTIME} cd ${RUNTIME} -python \ - ${WORKSPACE}/water-bodies/command-line-tools/crop/app.py \ +crop \ --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ --aoi "-121.399,39.834,-120.74,40.472" \ - --epsg "EPSG:4326" \ + --epsg "4326" \ --band nir \ No newline at end of file diff --git a/scripts/norm_diff.sh b/scripts/norm_diff.sh index a7b3dba..479ab7d 100644 --- a/scripts/norm_diff.sh +++ b/scripts/norm_diff.sh @@ -3,7 +3,6 @@ export RUNTIME=${WORKSPACE}/runs mkdir -p ${RUNTIME} cd ${RUNTIME} -python \ - ${WORKSPACE}/water-bodies/command-line-tools/norm_diff/app.py \ - crop_green.tif \ - crop_nir.tif \ No newline at end of file +norm_diff \ + --rasters crop_green.tif \ + --rasters crop_nir.tif \ No newline at end of file diff --git a/scripts/norm_diff_env.sh b/scripts/norm_diff_env.sh index 3a38f92..d20e562 100644 --- a/scripts/norm_diff_env.sh +++ b/scripts/norm_diff_env.sh @@ -1,3 +1,3 @@ python -m venv env_norm_diff source env_norm_diff/bin/activate -pip install --no-cache-dir rasterio click loguru +python -m pip install -e "${WORKSPACE}/water-bodies/command-line-tools/norm_diff" diff --git a/scripts/otsu.sh b/scripts/otsu.sh index f5ce1f0..19875cd 100644 --- a/scripts/otsu.sh +++ b/scripts/otsu.sh @@ -3,6 +3,5 @@ export RUNTIME=${WORKSPACE}/runs mkdir -p ${RUNTIME} cd ${RUNTIME} -python \ - ${WORKSPACE}/water-bodies/command-line-tools/otsu/app.py \ - norm_diff.tif \ No newline at end of file +otsu \ + --raster norm_diff.tif \ No newline at end of file diff --git a/scripts/otsu_env.sh b/scripts/otsu_env.sh index 61a860c..9a14ec8 100644 --- a/scripts/otsu_env.sh +++ b/scripts/otsu_env.sh @@ -1,3 +1,3 @@ python -m venv env_otsu source env_otsu/bin/activate -pip install --no-cache-dir rasterio scikit-image click loguru +python -m pip install -e "${WORKSPACE}/water-bodies/command-line-tools/otsu" diff --git a/scripts/podman-crop-green.sh b/scripts/podman-crop-green.sh index 2e0a035..6c8ad87 100644 --- a/scripts/podman-crop-green.sh +++ b/scripts/podman-crop-green.sh @@ -4,19 +4,16 @@ podman run \ --mount=type=bind,source=/workspace/mastering-app-package/runs,target=/runs \ --workdir=/runs \ --read-only=true \ - --user=1001:100 \ + --user="$(id -u):$(id -g)" \ --rm \ --env=HOME=/runs \ - --env=PYTHONPATH=/app \ localhost/crop:latest \ - python \ - -m \ - app \ + crop \ --aoi \ "-121.399,39.834,-120.74,40.472" \ --band \ green \ --epsg \ - "EPSG:4326" \ + "4326" \ --input-item \ https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A \ No newline at end of file diff --git a/scripts/podman-crop-nir.sh b/scripts/podman-crop-nir.sh index e8bf614..897292b 100644 --- a/scripts/podman-crop-nir.sh +++ b/scripts/podman-crop-nir.sh @@ -4,19 +4,16 @@ podman run \ --mount=type=bind,source=/workspace/mastering-app-package/runs,target=/runs \ --workdir=/runs \ --read-only=true \ - --user=1001:100 \ + --user="$(id -u):$(id -g)" \ --rm \ --env=HOME=/runs \ - --env=PYTHONPATH=/app \ localhost/crop:latest \ - python \ - -m \ - app \ + crop \ --aoi \ "-121.399,39.834,-120.74,40.472" \ --band \ nir \ --epsg \ - "EPSG:4326" \ + "4326" \ --input-item \ https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A \ No newline at end of file diff --git a/scripts/podman-norm-diff.sh b/scripts/podman-norm-diff.sh index 0312312..ab0940e 100644 --- a/scripts/podman-norm-diff.sh +++ b/scripts/podman-norm-diff.sh @@ -7,13 +7,10 @@ podman \ --mount=type=bind,source=/workspace/mastering-app-package/runs/crop_nir.tif,target=/inputs/crop_nir.tif,readonly \ --workdir=/runs \ --read-only=true \ - --user=1001:100 \ + --user="$(id -u):$(id -g)" \ --rm \ --env=HOME=/runs \ - --env=PYTHONPATH=/app \ localhost/norm-diff:latest \ - python \ - -m \ - app \ - /inputs/crop_green.tif \ - /inputs/crop_nir.tif \ No newline at end of file + norm_diff \ + --rasters /inputs/crop_green.tif \ + --rasters /inputs/crop_nir.tif \ No newline at end of file diff --git a/scripts/podman-otsu.sh b/scripts/podman-otsu.sh index 7667c6c..82b9beb 100644 --- a/scripts/podman-otsu.sh +++ b/scripts/podman-otsu.sh @@ -6,12 +6,9 @@ podman \ --mount=type=bind,source=/workspace/mastering-app-package/runs/norm_diff.tif,target=/inputs/norm_diff.tif,readonly \ --workdir=/runs \ --read-only=true \ - --user=1001:100 \ + --user="$(id -u):$(id -g)" \ --rm \ --env=HOME=/runs \ - --env=PYTHONPATH=/app \ localhost/otsu:latest \ - python \ - -m \ - app \ - /inputs/norm_diff.tif \ No newline at end of file + otsu \ + --raster /inputs/norm_diff.tif \ No newline at end of file diff --git a/scripts/podman-stac.sh b/scripts/podman-stac.sh index e371bea..6c37adf 100644 --- a/scripts/podman-stac.sh +++ b/scripts/podman-stac.sh @@ -6,15 +6,12 @@ podman \ --mount=type=bind,source=/workspace/mastering-app-package/runs/otsu.tif,target=/inputs/otsu.tif,readonly \ --workdir=/runs \ --read-only=true \ - --user=1001:100 \ + --user="$(id -u):$(id -g)" \ --rm \ --env=HOME=/runs \ - --env=PYTHONPATH=/app \ localhost/stac:latest \ - python \ - -m \ - app \ - --input-item \ + stac \ + --item \ https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A \ - --water-body \ + --rasters \ /inputs/otsu.tif \ No newline at end of file diff --git a/scripts/stac.sh b/scripts/stac.sh index 53a6e75..3838421 100644 --- a/scripts/stac.sh +++ b/scripts/stac.sh @@ -3,7 +3,6 @@ export RUNTIME=${WORKSPACE}/runs mkdir -p ${RUNTIME} cd ${RUNTIME} -python \ - ${WORKSPACE}/water-bodies/command-line-tools/stac/app.py \ - --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --water-body otsu.tif \ No newline at end of file +stac \ + --item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ + --rasters otsu.tif \ No newline at end of file diff --git a/scripts/stac_env.sh b/scripts/stac_env.sh index 9b22940..71078fe 100644 --- a/scripts/stac_env.sh +++ b/scripts/stac_env.sh @@ -1,3 +1,3 @@ python -m venv env_stac source env_stac/bin/activate -pip install --no-cache-dir pystac rio_stac loguru click +python -m pip install -e "${WORKSPACE}/water-bodies/command-line-tools/stac" From 2c7df9952c136e7f3035357ba5cb21c5a7f64467 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:42:49 +0200 Subject: [PATCH 12/23] feat(staging): package stage-in and stage-out commands Replace embedded staging scripts with installed Python commands and dedicated multistage images while preserving CWL job inputs and legacy released examples. Fail incomplete downloads and retain destination prefixes for absolute asset paths. --- .gitignore | 3 +- cwl-cli/stage-in.cwl | 54 +----- cwl-cli/stage-out.cwl | 95 +--------- scripts/build-stage-container.sh | 6 +- scripts/build_containers.sh | 4 +- .../command-line-tools/stage-in/Dockerfile | 23 +++ .../command-line-tools/stage-in/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/stage-in/README.md | 51 ++++++ .../stage-in/pyproject.toml | 142 +++++++++++++++ .../stage-in/src/stage/__about__.py | 1 + .../stage-in/src/stage/__init__.py | 1 + .../stage-in/src/stage/stage_in_cli.py | 12 ++ .../stage-in/src/stage/stage_in_impl.py | 43 +++++ .../stage-in/tests/__init__.py | 0 .../stage-in/tests/test_stage.py | 10 + .../stage-in/tests/test_stage_in.py | 67 +++++++ .../command-line-tools/stage-out/Dockerfile | 23 +++ .../command-line-tools/stage-out/LICENSE.md | 171 ++++++++++++++++++ .../command-line-tools/stage-out/README.md | 26 +++ .../stage-out/pyproject.toml | 145 +++++++++++++++ .../stage-out/src/stage_out/__about__.py | 1 + .../stage-out/src/stage_out/__init__.py | 1 + .../stage-out/src/stage_out/cli.py | 30 +++ .../stage-out/src/stage_out/stage_out_impl.py | 85 +++++++++ .../stage-out/tests/test_stage_out.py | 95 ++++++++++ 25 files changed, 1118 insertions(+), 142 deletions(-) create mode 100644 water-bodies/command-line-tools/stage-in/Dockerfile create mode 100644 water-bodies/command-line-tools/stage-in/LICENSE.md create mode 100644 water-bodies/command-line-tools/stage-in/README.md create mode 100644 water-bodies/command-line-tools/stage-in/pyproject.toml create mode 100644 water-bodies/command-line-tools/stage-in/src/stage/__about__.py create mode 100644 water-bodies/command-line-tools/stage-in/src/stage/__init__.py create mode 100644 water-bodies/command-line-tools/stage-in/src/stage/stage_in_cli.py create mode 100644 water-bodies/command-line-tools/stage-in/src/stage/stage_in_impl.py create mode 100644 water-bodies/command-line-tools/stage-in/tests/__init__.py create mode 100644 water-bodies/command-line-tools/stage-in/tests/test_stage.py create mode 100644 water-bodies/command-line-tools/stage-in/tests/test_stage_in.py create mode 100644 water-bodies/command-line-tools/stage-out/Dockerfile create mode 100644 water-bodies/command-line-tools/stage-out/LICENSE.md create mode 100644 water-bodies/command-line-tools/stage-out/README.md create mode 100644 water-bodies/command-line-tools/stage-out/pyproject.toml create mode 100644 water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py create mode 100644 water-bodies/command-line-tools/stage-out/src/stage_out/__init__.py create mode 100644 water-bodies/command-line-tools/stage-out/src/stage_out/cli.py create mode 100644 water-bodies/command-line-tools/stage-out/src/stage_out/stage_out_impl.py create mode 100644 water-bodies/command-line-tools/stage-out/tests/test_stage_out.py diff --git a/.gitignore b/.gitignore index 2b01dc2..9b830c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__/ *.pyo *.pyd *.whl -*.puml \ No newline at end of file +*.puml +dist/ diff --git a/cwl-cli/stage-in.cwl b/cwl-cli/stage-in.cwl index a101cd7..abd39ae 100644 --- a/cwl-cli/stage-in.cwl +++ b/cwl-cli/stage-in.cwl @@ -38,6 +38,8 @@ inputs: type: string label: STAC item reference doc: URI of the STAC item whose assets are downloaded. + inputBinding: + prefix: --reference outputs: staged: type: Directory @@ -45,55 +47,13 @@ outputs: glob: . label: Staged acquisition doc: Directory containing the staged STAC catalog, item, and downloaded assets. -baseCommand: - - python - - stage.py -arguments: - - $( inputs.reference ) +baseCommand: stage-in requirements: DockerRequirement: - dockerPull: localhost/stage:latest + dockerPull: localhost/stage-in:latest InlineJavascriptRequirement: {} NetworkAccess: networkAccess: true - InitialWorkDirRequirement: - listing: - - entryname: stage.py - entry: |- - import pystac - import stac_asset - import asyncio - import os - import sys - - config = stac_asset.Config(warn=True) - - async def main(href: str): - - item = pystac.read_file(href) - - os.makedirs(item.id, exist_ok=True) - cwd = os.getcwd() - - os.chdir(item.id) - item = await stac_asset.download_item(item=item, directory=".", config=config) - os.chdir(cwd) - - cat = pystac.Catalog( - id="catalog", - description=f"catalog with staged {item.id}", - title=f"catalog with staged {item.id}", - ) - cat.add_item(item) - - cat.normalize_hrefs("./") - cat.save(catalog_type=pystac.CatalogType.SELF_CONTAINED) - - return cat - - href = sys.argv[1] - - cat = asyncio.run(main(href)) - - - + EnvVarRequirement: + envDef: + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/cwl-cli/stage-out.cwl b/cwl-cli/stage-out.cwl index aa5f20a..a0e3711 100644 --- a/cwl-cli/stage-out.cwl +++ b/cwl-cli/stage-out.cwl @@ -72,16 +72,14 @@ outputs: type: string label: Uploaded STAC catalog doc: S3 URI of the uploaded results catalog. -baseCommand: - - python - - stage.py +baseCommand: stage-out arguments: - $( inputs.stac_catalog.path ) - $( inputs.s3_bucket ) - $( inputs.sub_path ) requirements: DockerRequirement: - dockerPull: ghcr.io/terradue/ogc-eo-application-package-hands-on/stage:1.3.2 + dockerPull: localhost/stage-out:latest InlineJavascriptRequirement: {} NetworkAccess: networkAccess: true @@ -91,92 +89,5 @@ requirements: aws_secret_access_key: $( inputs.aws_secret_access_key ) aws_region_name: $( inputs.region_name ) aws_endpoint_url: $( inputs.endpoint_url ) + PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: {} - InitialWorkDirRequirement: - listing: - - entryname: stage.py - entry: |- - import os - import sys - import pystac - import botocore - import boto3 - import shutil - from pystac.stac_io import DefaultStacIO, StacIO - from urllib.parse import urlparse - - cat_url = sys.argv[1] - bucket = sys.argv[2] - subfolder = sys.argv[3] - - aws_access_key_id = os.environ["aws_access_key_id"] - aws_secret_access_key = os.environ["aws_secret_access_key"] - region_name = os.environ["aws_region_name"] - endpoint_url = os.environ["aws_endpoint_url"] - - shutil.copytree(cat_url, "/tmp/catalog") - cat = pystac.read_file(os.path.join("/tmp/catalog", "catalog.json")) - - class CustomStacIO(DefaultStacIO): - """Custom STAC IO class that uses boto3 to read from S3.""" - - def __init__(self): - self.session = botocore.session.Session() - self.s3_client = self.session.create_client( - service_name="s3", - use_ssl=True, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - region_name=region_name, - ) - - def write_text(self, dest, txt, *args, **kwargs): - parsed = urlparse(dest) - if parsed.scheme == "s3": - self.s3_client.put_object( - Body=txt.encode("UTF-8"), - Bucket=parsed.netloc, - Key=parsed.path[1:], - ContentType="application/geo+json", - ) - else: - super().write_text(dest, txt, *args, **kwargs) - - - client = boto3.client( - "s3", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - region_name=region_name, - ) - - StacIO.set_default(CustomStacIO) - - for item in cat.get_items(): - for key, asset in item.get_assets().items(): - s3_path = os.path.normpath( - os.path.join(os.path.join(subfolder, item.id, asset.href)) - ) - print(f"upload {asset.href} to s3://{bucket}/{s3_path}",file=sys.stderr) - client.upload_file( - asset.get_absolute_href(), - bucket, - s3_path, - ) - asset.href = f"s3://{bucket}/{s3_path}" - item.add_asset(key, asset) - - cat.normalize_hrefs(f"s3://{bucket}/{subfolder}") - - for item in cat.get_items(): - # upload item to S3 - print(f"upload {item.id} to s3://{bucket}/{subfolder}", file=sys.stderr) - pystac.write_file(item, item.get_self_href()) - - # upload catalog to S3 - print(f"upload catalog.json to s3://{bucket}/{subfolder}", file=sys.stderr) - pystac.write_file(cat, cat.get_self_href()) - - print(f"s3://{bucket}/{subfolder}/catalog.json", file=sys.stdout) diff --git a/scripts/build-stage-container.sh b/scripts/build-stage-container.sh index 2aab21c..cfa8aa4 100644 --- a/scripts/build-stage-container.sh +++ b/scripts/build-stage-container.sh @@ -4,4 +4,8 @@ podman \ build \ --format docker \ -t localhost/stage:latest \ - ${WORKSPACE}/water-bodies/command-line-tools/stage \ No newline at end of file + ${WORKSPACE}/water-bodies/command-line-tools/stage + +# Current checkout uses separate installed staging commands. +podman build --format docker -t localhost/stage-in:latest ${WORKSPACE}/water-bodies/command-line-tools/stage-in +podman build --format docker -t localhost/stage-out:latest ${WORKSPACE}/water-bodies/command-line-tools/stage-out diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh index c46af80..611b82d 100644 --- a/scripts/build_containers.sh +++ b/scripts/build_containers.sh @@ -3,4 +3,6 @@ export WORKSPACE=/workspace/mastering-app-package podman build --format docker -t localhost/crop:latest ${WORKSPACE}/water-bodies/command-line-tools/crop podman build --format docker -t localhost/norm-diff:latest ${WORKSPACE}/water-bodies/command-line-tools/norm_diff podman build --format docker -t localhost/otsu:latest ${WORKSPACE}/water-bodies/command-line-tools/otsu -podman build --format docker -t localhost/stac:latest ${WORKSPACE}/water-bodies/command-line-tools/stac \ No newline at end of file +podman build --format docker -t localhost/stac:latest ${WORKSPACE}/water-bodies/command-line-tools/stac +podman build --format docker -t localhost/stage-in:latest ${WORKSPACE}/water-bodies/command-line-tools/stage-in +podman build --format docker -t localhost/stage-out:latest ${WORKSPACE}/water-bodies/command-line-tools/stage-out diff --git a/water-bodies/command-line-tools/stage-in/Dockerfile b/water-bodies/command-line-tools/stage-in/Dockerfile new file mode 100644 index 0000000..833d625 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/Dockerfile @@ -0,0 +1,23 @@ +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . + +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime +RUN apt-get update && \ + apt-get install -y --no-install-recommends libexpat1 && \ + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + stage-in --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work +ENTRYPOINT [] diff --git a/water-bodies/command-line-tools/stage-in/LICENSE.md b/water-bodies/command-line-tools/stage-in/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/stage-in/README.md b/water-bodies/command-line-tools/stage-in/README.md new file mode 100644 index 0000000..53036ac --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/README.md @@ -0,0 +1,51 @@ +# Stage in + +Stage a STAC item and its assets into the current directory. The `stage-in` command writes `catalog.json` and an item directory containing the downloaded assets, matching the output of `cwl-cli/stage-in.cwl`. + +## Hatch setup + +From the repository root: + +```bash +cd water-bodies/command-line-tools/stage-in +hatch env create default +source "$(hatch env find default)/bin/activate" +``` + +Python 3.12 or newer and Hatch are required. Run the command from the directory where outputs should be written: + +```bash +stage-in --reference /path/to/item.json +# Remote STAC item URLs are also accepted. +stage-in --help +``` + +Run `deactivate` when finished. Progress and failures are logged by Loguru for the local processing tools. + +## Checks and tests + +From this package directory: + +```bash +hatch run test:test +``` + +The Ruff complexity limit is 4. Tests use local fixtures rather than remote imagery. + +## Container + +From the repository root: + +```bash +podman build -t localhost/stage-in:latest water-bodies/command-line-tools/stage-in +``` + +The Dockerfile builds a wheel with Hatchling through pip and installs it in a non-root runtime environment. + +## License + +See [LICENSE.md](LICENSE.md). + +Alternatively, install with `python -m pip install -e .` in a Python 3.12 or newer virtual environment. The container runs as UID/GID 2000, with `stage-in` installed in `/app/venv/bin`. + +Failed asset downloads fail the command before catalog publication. HTTP requests have a 60-second timeout and up to three attempts. diff --git a/water-bodies/command-line-tools/stage-in/pyproject.toml b/water-bodies/command-line-tools/stage-in/pyproject.toml new file mode 100644 index 0000000..2dac65c --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/pyproject.toml @@ -0,0 +1,142 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stage" +dynamic = ["version"] +description = "Stage STAC items and assets into a local catalog" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "stac-asset==0.4.7", + "click==8.5.0", + "pystac==1.15.2", + "boto3==1.40.18", + "loguru==0.7.3", +] + +[project.scripts] +stage-in = "stage.stage_in_cli:stage_in" + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/stage-in/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/stage/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/stage tests}" + +[tool.coverage.run] +source_pkgs = ["stage", "tests"] +branch = true +parallel = true +omit = [ + "src/stage/__about__.py", +] + +[tool.coverage.paths] +stage = ["src/stage", "*/stage/src/stage"] +tests = ["tests", "*/stage/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=stage --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stage-in/src/stage/__about__.py b/water-bodies/command-line-tools/stage-in/src/stage/__about__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/src/stage/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/stage-in/src/stage/__init__.py b/water-bodies/command-line-tools/stage-in/src/stage/__init__.py new file mode 100644 index 0000000..73dec11 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/src/stage/__init__.py @@ -0,0 +1 @@ +"""Environment for staging STAC assets.""" diff --git a/water-bodies/command-line-tools/stage-in/src/stage/stage_in_cli.py b/water-bodies/command-line-tools/stage-in/src/stage/stage_in_cli.py new file mode 100644 index 0000000..ba36e7d --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/src/stage/stage_in_cli.py @@ -0,0 +1,12 @@ +"""Stage-in command-line interface.""" + +import click + +from stage.stage_in_impl import execute + +stage_in = click.Command( + name="stage-in", + callback=execute, + help="Download a STAC item and its assets into a self-contained catalog.", + params=[click.Option(["--reference"], required=True, help="STAC item URL or local path.")], +) diff --git a/water-bodies/command-line-tools/stage-in/src/stage/stage_in_impl.py b/water-bodies/command-line-tools/stage-in/src/stage/stage_in_impl.py new file mode 100644 index 0000000..5412a68 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/src/stage/stage_in_impl.py @@ -0,0 +1,43 @@ +"""Download a STAC item and package its assets in a local catalog.""" + +import asyncio +from pathlib import Path + +import click +import pystac +import stac_asset +from loguru import logger +from stac_asset.errors import AssetOverwriteError, DownloadError + + +async def stage_item(reference: str) -> pystac.Catalog: + """Stage an item's assets under its ID and write catalog.json.""" + logger.info("Reading input STAC item") + item = pystac.read_file(reference) + if not isinstance(item, pystac.Item): + raise ValueError("Reference must point to a STAC item") + directory = Path(item.id) + if directory.name != item.id or item.id in {"", ".", ".."}: + raise ValueError("STAC item id must be a single directory name") + directory.mkdir(exist_ok=True) + logger.info("Downloading assets for {} into {}", item.id, directory.resolve()) + item = await stac_asset.download_item(item=item, directory=directory.resolve(), config=stac_asset.Config(http_client_timeout=60, http_max_attempts=3)) + catalog = pystac.Catalog( + id="catalog", + description=f"catalog with staged {item.id}", + title=f"catalog with staged {item.id}", + ) + catalog.add_item(item) + catalog.normalize_hrefs("./") + catalog.save(catalog_type=pystac.CatalogType.SELF_CONTAINED) + logger.success("Wrote catalog.json for {}", item.id) + return catalog + + +def execute(*, reference: str) -> None: + """Synchronous callback for the stage-in console command.""" + try: + asyncio.run(stage_item(reference)) + except (ValueError, OSError, pystac.STACError, DownloadError, AssetOverwriteError) as exc: + logger.error("Stage-in failed: {}", exc) + raise click.ClickException(str(exc)) from exc diff --git a/water-bodies/command-line-tools/stage-in/tests/__init__.py b/water-bodies/command-line-tools/stage-in/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/water-bodies/command-line-tools/stage-in/tests/test_stage.py b/water-bodies/command-line-tools/stage-in/tests/test_stage.py new file mode 100644 index 0000000..794b880 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/tests/test_stage.py @@ -0,0 +1,10 @@ +"""Verify the installed staging console entry point.""" +from importlib.metadata import distribution +from click.testing import CliRunner + + +def test_stage_in_cli(): + entry = next(e for e in distribution('stage').entry_points if e.name == 'stage-in') + result = CliRunner().invoke(entry.load(), ['--help']) + assert result.exit_code == 0, result.output + assert '--reference' in result.output diff --git a/water-bodies/command-line-tools/stage-in/tests/test_stage_in.py b/water-bodies/command-line-tools/stage-in/tests/test_stage_in.py new file mode 100644 index 0000000..c91df24 --- /dev/null +++ b/water-bodies/command-line-tools/stage-in/tests/test_stage_in.py @@ -0,0 +1,67 @@ +"""Test stage-in with local STAC assets.""" + +from datetime import datetime, timezone +from pathlib import Path + +import pystac +from click.testing import CliRunner + +from stage.stage_in_cli import stage_in + + +def test_stages_local_item(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + asset = source / "data.txt" + asset.write_text("local asset contents") + item = pystac.Item( + "scene", + geometry=None, + bbox=None, + datetime=datetime(2026, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + item.add_asset("data", pystac.Asset("./data.txt", roles=["data"])) + item.set_self_href(str(source / "item.json")) + item.save_object() + output = tmp_path / "output" + output.mkdir() + monkeypatch.chdir(output) + result = CliRunner().invoke(stage_in, ["--reference", str(source / "item.json")]) + assert result.exit_code == 0, result.output + assert Path.cwd() == output + catalog = pystac.Catalog.from_file(str(output / "catalog.json")) + staged = next(catalog.get_items()) + assert staged.id == "scene" + local_asset = Path(staged.assets["data"].get_absolute_href()) + assert local_asset.is_relative_to(output / "scene") + assert local_asset.read_text() == "local asset contents" + assert not Path(staged.assets["data"].href).is_absolute() + + +def test_rejects_catalog(tmp_path, monkeypatch): + catalog = pystac.Catalog("empty", "Empty catalog") + catalog.set_self_href(str(tmp_path / "source.json")) + catalog.save_object() + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(stage_in, ["--reference", str(tmp_path / "source.json")]) + assert result.exit_code == 1 + assert "must point to a STAC item" in result.output + assert not (tmp_path / "catalog.json").exists() + + +def test_reference_required(): + result = CliRunner().invoke(stage_in, []) + assert result.exit_code == 2 + assert "--reference" in result.output + + +def test_failed_download_does_not_publish_catalog(tmp_path, monkeypatch): + item = pystac.Item("missing", None, None, datetime(2026, 1, 1, tzinfo=timezone.utc), {}) + item.add_asset("data", pystac.Asset(str(tmp_path / "absent.tif"))) + source = tmp_path / "source.json" + item.save_object(dest_href=str(source)) + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(stage_in, ["--reference", str(source)]) + assert result.exit_code == 1 + assert not (tmp_path / "catalog.json").exists() diff --git a/water-bodies/command-line-tools/stage-out/Dockerfile b/water-bodies/command-line-tools/stage-out/Dockerfile new file mode 100644 index 0000000..836365d --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/Dockerfile @@ -0,0 +1,23 @@ +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS builder +WORKDIR /src +COPY pyproject.toml README.md LICENSE.md ./ +COPY src/ ./src/ +RUN python -m pip wheel --no-cache-dir --no-deps --wheel-dir /dist . + +FROM docker.io/library/python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1 AS runtime +RUN apt-get update && \ + apt-get install -y --no-install-recommends libexpat1 && \ + rm -rf /var/lib/apt/lists/* && \ + groupadd --gid 2000 neo && \ + useradd --uid 2000 --gid 2000 --create-home neo && \ + python -m venv /app/venv && \ + mkdir -p /work && chown 2000:2000 /work +ENV VIRTUAL_ENV=/app/venv +ENV PATH="/app/venv/bin:$PATH" +COPY --from=builder /dist/ /tmp/wheels/ +RUN pip install --no-cache-dir /tmp/wheels/*.whl && \ + stage-out --help && \ + rm -rf /tmp/wheels +USER 2000:2000 +WORKDIR /work +ENTRYPOINT [] diff --git a/water-bodies/command-line-tools/stage-out/LICENSE.md b/water-bodies/command-line-tools/stage-out/LICENSE.md new file mode 100644 index 0000000..ab06007 --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/LICENSE.md @@ -0,0 +1,171 @@ +# Creative Commons Attribution-ShareAlike 4.0 International + +Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +**Using Creative Commons Public Licenses** + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +* __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). + +* __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). + +## Creative Commons Attribution-ShareAlike 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +### Section 1 – Definitions. + +a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. __BY-SA Compatible License__ means a license listed at [creativecommons.org/compatiblelicenses](http://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. + +d. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + +e. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +f. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +g. __License Elements__ means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution and ShareAlike. + +h. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +i. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +j. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. + +k. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +l. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +m. __You__ means the individual or entity exercising the Licensed Rights under this Public License. __Your__ has a corresponding meaning. + +### Section 2 – Scope. + +a. ___License grant.___ + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. __Term.__ The term of this Public License is specified in Section 6(a). + + 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. __Downstream recipients.__ + + A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. __Additional offer from the Licensor – Adapted Material.__ Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. + + C. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. ___Other rights.___ + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +### Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. ___Attribution.___ + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + +b. ___ShareAlike.___ + +In addition to the conditions in Section 3(a), if You Share Adapted Material You produce, the following conditions also apply. + +1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-SA Compatible License. + +2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. + +3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. + +### Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section 3(b); and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +### Section 5 – Disclaimer of Warranties and Limitation of Liability. + +a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ + +b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +### Section 6 – Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +### Section 7 – Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +### Section 8 – Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +> Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [CC0 Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. +> +> Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/water-bodies/command-line-tools/stage-out/README.md b/water-bodies/command-line-tools/stage-out/README.md new file mode 100644 index 0000000..919821a --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/README.md @@ -0,0 +1,26 @@ +# Stage out + +Upload a local STAC catalog and its assets to S3. The input directory is copied +to a temporary directory before links are rewritten; the original stays intact. + +From this directory: + +```bash +hatch run stage-out --help +hatch run stage-out /path/to/catalog bucket-name results/run-1 +``` + +Set `aws_access_key_id`, `aws_secret_access_key`, `aws_region_name`, and +`aws_endpoint_url` in the environment. Credentials are never command-line arguments. +The command prints the uploaded catalog's S3 URL to stdout and logs to stderr. + +```bash +hatch run test:test +hatch build +podman build -t localhost/stage-out:latest . +``` + +`cwl-cli/stage-out.cwl` invokes this command with the same positional arguments +and environment settings. Build the image before running that CWL tool. + +Python 3.12 or newer is required. Alternatively, install with `python -m pip install -e .` in a virtual environment. The multistage image installs a Hatchling-built wheel in `/app/venv` and runs as UID/GID 2000. Licensed under [CC-BY-SA-4.0](LICENSE.md). diff --git a/water-bodies/command-line-tools/stage-out/pyproject.toml b/water-bodies/command-line-tools/stage-out/pyproject.toml new file mode 100644 index 0000000..77bda22 --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/pyproject.toml @@ -0,0 +1,145 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stage-out" +dynamic = ["version"] +description = "Upload STAC catalogs and assets to S3" +readme = "README.md" +requires-python = ">=3.12" +license = "CC-BY-SA-4.0" +keywords = [] +authors = [ + { name = "Fabrice Brito", email = "info@terradue.com" }, + { name = "Simone Tripodi", email = "info@terradue.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "click==8.5.0", + "pystac==1.15.2", + "boto3==1.40.18", + "loguru==0.7.3", +] + +[project.scripts] +stage-out = "stage_out.cli:stage_out" + +[project.urls] +Documentation = "https://github.com/eoap/mastering-app-package/blob/main/water-bodies/command-line-tools/stage-out/README.md" +Issues = "https://github.com/eoap/mastering-app-package/issues" +Source = "https://github.com/eoap/mastering-app-package" + +[tool.hatch.version] +path = "src/stage_out/__about__.py" + +[tool.hatch.envs.types] +extra-dependencies = [ + "mypy==1.19.1", +] + + +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/stage_out tests}" + +[tool.coverage.run] +source_pkgs = ["stage_out", "tests"] +branch = true +parallel = true +omit = [ + "src/stage_out/__about__.py", +] + +[tool.coverage.paths] +stage_out = ["src/stage_out", "*/stage-out/src/stage_out"] +tests = ["tests", "*/stage-out/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.hatch.envs.test] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "pytest-cov>=6.2,<8", +] + +[tool.hatch.envs.test.scripts] +test = "pytest {args:tests}" +testv = "pytest --verbose {args:tests}" +test-cov = "pytest --cov=stage_out --cov-report=term-missing {args:tests}" +test-show-setup = "pytest --setup-show" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = [ + "--strict-config", + "--strict-markers", +] + +[[tool.hatch.envs.test.matrix]] +python = ["3.12", "3.13", "3.14"] + +[tool.hatch.envs.default] +skip-install = false +extra-dependencies = [ + "bash-kernel>=0.10,<1", +] + +[tool.hatch.envs.dev] +skip-install = false +dependencies = [ + "pytest>=8.4,<10", + "mypy>=2.3.0", + "ruff>=0.16.0", + "bandit>=1.9.4", +] + +[tool.ruff.lint] +select = [ + "F", # Pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "FA", # flake8-future-annotations + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "C90", # McCabe complexity +] +ignore = [ + "E501", # Line length handled by formatter + "W191", # Indentation handled by formatter + "E111", + "E114", + "E117", +] + +# Preserve the environment variable names in the existing CWL interface. +[tool.ruff.lint.per-file-ignores] +"src/stage_out/stage_out_impl.py" = ["SIM112"] + +[tool.ruff.lint.mccabe] +max-complexity = 4 + +[tool.hatch.envs.dev.scripts] +typecheck = "mypy --install-types --non-interactive {args:src tests}" +lint = "ruff format {args:src tests}" +check = "ruff check --fix {args:src tests}" +security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py b/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/water-bodies/command-line-tools/stage-out/src/stage_out/__init__.py b/water-bodies/command-line-tools/stage-out/src/stage_out/__init__.py new file mode 100644 index 0000000..fc0ad8d --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/src/stage_out/__init__.py @@ -0,0 +1 @@ +"""Upload STAC catalogs and their assets to S3.""" diff --git a/water-bodies/command-line-tools/stage-out/src/stage_out/cli.py b/water-bodies/command-line-tools/stage-out/src/stage_out/cli.py new file mode 100644 index 0000000..4b6a59c --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/src/stage_out/cli.py @@ -0,0 +1,30 @@ +"""Command-line entry point for staging results to S3.""" + +from pathlib import Path + +import click +import pystac +from botocore.exceptions import BotoCoreError, ClientError + +from stage_out.stage_out_impl import execute + + +@click.command() +@click.argument( + "catalog", type=click.Path(exists=True, file_okay=False, path_type=Path) +) +@click.argument("bucket") +@click.argument("subfolder") +def stage_out(catalog: Path, bucket: str, subfolder: str) -> None: + """Upload CATALOG and its assets to BUCKET under SUBFOLDER. + + Credentials and endpoint settings come from the aws_access_key_id, + aws_secret_access_key, aws_region_name and aws_endpoint_url environment variables. + """ + try: + result = execute(catalog, bucket, subfolder) + except KeyError as exc: + raise click.ClickException(f"Missing environment setting: {exc.args[0]}") from exc + except (ValueError, OSError, pystac.STACError, BotoCoreError, ClientError) as exc: + raise click.ClickException(str(exc)) from exc + click.echo(result) diff --git a/water-bodies/command-line-tools/stage-out/src/stage_out/stage_out_impl.py b/water-bodies/command-line-tools/stage-out/src/stage_out/stage_out_impl.py new file mode 100644 index 0000000..5d1d4c8 --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/src/stage_out/stage_out_impl.py @@ -0,0 +1,85 @@ +"""Stage a local STAC catalog and its assets to S3.""" + +from __future__ import annotations + +import os +import posixpath +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any +from urllib.parse import urlparse + +import boto3 +import pystac +from loguru import logger +from pystac.stac_io import DefaultStacIO + + +class S3StacIO(DefaultStacIO): + """Write STAC documents using the configured S3 client.""" + + def __init__(self, client: Any) -> None: + super().__init__() + self.client = client + + def write_text(self, dest: Any, txt: str, *args: Any, **kwargs: Any) -> None: + parsed = urlparse(str(dest)) + if parsed.scheme == "s3": + self.client.put_object( + Body=txt.encode("UTF-8"), + Bucket=parsed.netloc, + Key=parsed.path.lstrip("/"), + ContentType="application/geo+json", + ) + else: + super().write_text(dest, txt, *args, **kwargs) + + +def upload_assets(item: pystac.Item, client: Any, bucket: str, subfolder: str) -> None: + """Upload assets before rewriting their references.""" + if Path(item.id).name != item.id or item.id in {"", ".", ".."}: + raise ValueError("STAC item id must be a single directory name") + for asset in item.assets.values(): + parsed = urlparse(asset.href) + relative = posixpath.basename(parsed.path) if parsed.scheme or posixpath.isabs(parsed.path) else parsed.path + relative = posixpath.normpath(relative) + if relative == ".." or relative.startswith("../"): + relative = posixpath.basename(relative) + key = posixpath.normpath(posixpath.join(subfolder, item.id, relative)) + logger.info("Uploading asset to s3://{}/{}", bucket, key) + client.upload_file(asset.get_absolute_href(), bucket, key) + asset.href = f"s3://{bucket}/{key}" + + +def upload_catalog( + catalog: pystac.Catalog, client: Any, bucket: str, subfolder: str +) -> str: + """Upload assets and publish the rewritten catalog documents.""" + items = list(catalog.get_items(recursive=True)) + for item in items: + upload_assets(item, client, bucket, subfolder) + destination = f"s3://{bucket}/{subfolder}" + catalog.normalize_hrefs(destination) + stac_io = S3StacIO(client) + catalog.save(catalog_type=pystac.CatalogType.SELF_CONTAINED, stac_io=stac_io) + logger.info("Published catalog to {}/catalog.json", destination) + return f"{destination}/catalog.json" + + +def execute(catalog: Path, bucket: str, subfolder: str) -> str: + """Copy the input catalog, upload it, and return its S3 URL.""" + client = boto3.client( + "s3", + aws_access_key_id=os.environ["aws_access_key_id"], + aws_secret_access_key=os.environ["aws_secret_access_key"], + region_name=os.environ["aws_region_name"], + endpoint_url=os.environ["aws_endpoint_url"], + ) + with TemporaryDirectory(prefix="stage-out-") as directory: + copied = Path(directory) / "catalog" + shutil.copytree(catalog, copied) + loaded = pystac.read_file(str(copied / "catalog.json")) + if not isinstance(loaded, pystac.Catalog): + raise ValueError("The input must contain a STAC catalog or collection") + return upload_catalog(loaded, client, bucket, subfolder) diff --git a/water-bodies/command-line-tools/stage-out/tests/test_stage_out.py b/water-bodies/command-line-tools/stage-out/tests/test_stage_out.py new file mode 100644 index 0000000..9fe8fca --- /dev/null +++ b/water-bodies/command-line-tools/stage-out/tests/test_stage_out.py @@ -0,0 +1,95 @@ +import json +from datetime import datetime, timezone +from unittest.mock import Mock + +import pystac +import pytest +from click.testing import CliRunner + +from stage_out.cli import stage_out +from stage_out.stage_out_impl import execute + + +@pytest.fixture +def catalog_directory(tmp_path): + catalog = pystac.Catalog(id="catalog", description="Test catalog") + item = pystac.Item( + id="water", + geometry=None, + bbox=None, + datetime=datetime(2026, 1, 1, tzinfo=timezone.utc), + properties={}, + ) + item.add_asset("mask", pystac.Asset(href="mask.tif")) + catalog.add_item(item) + catalog.normalize_hrefs(str(tmp_path)) + catalog.save(catalog_type=pystac.CatalogType.SELF_CONTAINED) + (tmp_path / "water" / "mask.tif").write_bytes(b"raster data") + return tmp_path + + +@pytest.fixture +def client(monkeypatch): + for name in ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_region_name", + "aws_endpoint_url", + ): + monkeypatch.setenv(name, "test") + client = Mock() + monkeypatch.setattr( + "stage_out.stage_out_impl.boto3.client", lambda *a, **kw: client + ) + return client + + +def test_upload_preserves_input_and_publishes_links(catalog_directory, client): + original = (catalog_directory / "water" / "water.json").read_bytes() + result = execute(catalog_directory, "bucket", "results/run") + assert result == "s3://bucket/results/run/catalog.json" + assert client.upload_file.call_args.args[1:] == ( + "bucket", + "results/run/water/mask.tif", + ) + documents = { + call.kwargs["Key"]: json.loads(call.kwargs["Body"]) + for call in client.put_object.call_args_list + } + assert ( + documents["results/run/water/water.json"]["assets"]["mask"]["href"] + == "s3://bucket/results/run/water/mask.tif" + ) + assert "results/run/catalog.json" in documents + assert (catalog_directory / "water" / "water.json").read_bytes() == original + + +def test_cli(catalog_directory, client): + result = CliRunner().invoke(stage_out, [str(catalog_directory), "bucket", "run"]) + assert result.exit_code == 0, result.output + assert result.stdout.strip() == "s3://bucket/run/catalog.json" + + +def test_upload_failure_propagates(catalog_directory, client): + client.upload_file.side_effect = OSError("Upload failed") + with pytest.raises(OSError, match="Upload failed"): + execute(catalog_directory, "bucket", "run") + client.put_object.assert_not_called() + + +def test_absolute_asset_keeps_destination_prefix(catalog_directory, client): + item_path = catalog_directory / "water" / "water.json" + data = json.loads(item_path.read_text()) + data["assets"]["mask"]["href"] = str(catalog_directory / "water" / "mask.tif") + item_path.write_text(json.dumps(data)) + execute(catalog_directory, "bucket", "results/run") + assert client.upload_file.call_args.args[1:] == ( + "bucket", "results/run/water/mask.tif") + + +def test_cli_reports_upload_failure(catalog_directory, client): + client.upload_file.side_effect = OSError("Upload failed") + result = CliRunner().invoke(stage_out, [str(catalog_directory), "bucket", "run"]) + assert result.exit_code == 1 + assert "Error: Upload failed" in result.output + client.put_object.assert_not_called() From 1908ed45ab8576d96c20972f3b0e2c387d20d442 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:42:52 +0200 Subject: [PATCH 13/23] ci(staging): test and build staging packages and images Include stage-in and stage-out in package tests, wheel builds, container smoke checks, and the release image matrix. --- .github/workflows/build.yaml | 2 +- .github/workflows/processing-tests.yaml | 8 +++++--- docs/release/ci.md | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8963005..52d0ac2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -49,7 +49,7 @@ jobs: strategy: matrix: - step: [crop, norm_diff, otsu, stac, stage] + step: [crop, norm_diff, otsu, stac, stage, stage-in, stage-out] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/processing-tests.yaml b/.github/workflows/processing-tests.yaml index 99c1ea8..4dc8370 100644 --- a/.github/workflows/processing-tests.yaml +++ b/.github/workflows/processing-tests.yaml @@ -29,13 +29,15 @@ jobs: ./water-bodies/command-line-tools/crop \ ./water-bodies/command-line-tools/norm_diff \ ./water-bodies/command-line-tools/otsu \ - ./water-bodies/command-line-tools/stac + ./water-bodies/command-line-tools/stac \ + ./water-bodies/command-line-tools/stage-in \ + ./water-bodies/command-line-tools/stage-out - name: Test commands and processing chain - run: pytest --import-mode=importlib tests water-bodies/command-line-tools/{crop,norm_diff,otsu,stac}/tests + run: pytest --import-mode=importlib tests water-bodies/command-line-tools/{crop,norm_diff,otsu,stac,stage-in,stage-out}/tests - name: Validate CWL and build wheels and images run: | while IFS= read -r cwl; do cwltool --validate "$cwl"; done < <(git ls-files '*.cwl') - for package in crop norm_diff otsu stac; do + for package in crop norm_diff otsu stac stage-in stage-out; do python -m build --wheel --outdir /tmp/wheels "water-bodies/command-line-tools/$package" docker build -t "processing-$package:test" "water-bodies/command-line-tools/$package" docker run --rm "processing-$package:test" "$package" --help diff --git a/docs/release/ci.md b/docs/release/ci.md index 08342f5..21f31f1 100644 --- a/docs/release/ci.md +++ b/docs/release/ci.md @@ -53,7 +53,7 @@ Below an example of a GitHub CI configuration implementing the scenario: --8<-- ``` -The processing test workflow installs and tests the four Python packages, runs +The processing test workflow installs and tests the six Python packages, runs a synthetic processing chain, validates the CWL, and builds wheels and images. Release image digests are written to mandatory `requirements.DockerRequirement` entries. `scripts/pack-cwl-release.py` selects the workflow explicitly, embeds From b5275c04fda19af539dbb7ee58bc89891d3ce1bd Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:42:54 +0200 Subject: [PATCH 14/23] docs(labs): document packaged staging tools and images Update the staging notebook and documentation for installed commands, separate images, bounded downloads, and catalog uploads. --- docs/app/water-bodies-detection.md | 4 +- docs/cwl-workflow/stage-in.md | 45 +++++--------- docs/cwl-workflow/stage-out.md | 61 +++++++------------ .../4-CWL-Workflows/2.1-stage-in.ipynb | 6 +- 4 files changed, 43 insertions(+), 73 deletions(-) diff --git a/docs/app/water-bodies-detection.md b/docs/app/water-bodies-detection.md index 652ae79..daa1790 100644 --- a/docs/app/water-bodies-detection.md +++ b/docs/app/water-bodies-detection.md @@ -132,4 +132,6 @@ with `--userns=keep-id` so bind-mounted output folders remain writable. Rebuild the four local images before running the updated CWL documents; previous images do not contain the installed console commands. -Stage-in and stage-out retain their existing implementation. +Stage-in and stage-out are also installed Python packages. Their CWL tools invoke +`stage-in --reference ` and `stage-out ` +in separate images. The original `stage` image is retained for released examples. diff --git a/docs/cwl-workflow/stage-in.md b/docs/cwl-workflow/stage-in.md index 0b21711..40eab7e 100644 --- a/docs/cwl-workflow/stage-in.md +++ b/docs/cwl-workflow/stage-in.md @@ -10,43 +10,28 @@ This step has a dedicated lab available at /workspace/mastering-app-package/prac Below a `stage-in.cwl` CWL (Common Workflow Language) document for a command-line tool that executes a Python script. -The document is designed to use a container and execute a Python script named "stage.py": +The document invokes the installed `stage-in --reference ` command in +`localhost/stage-in:latest`. The reference remains a string, so existing job +documents continue to work. -```yaml linenums="1" hl_lines="26-59" title="stage-in.cwl" +```yaml linenums="1" title="stage-in.cwl" --8<-- cwl-cli/stage-in.cwl --8<-- ``` -The `stage.py` Python script uses the `pystac` and `stac_asset` libraries to stage STAC Items, download them, and create a STAC Catalog. -It's designed to be run from the command line, taking the URL of a STAC Item as a command-line argument. +The Python package reads a STAC Item, downloads its assets with `stac-asset`, +and writes a self-contained `catalog.json` with relative asset links. It keeps +the working directory unchanged and validates that the item ID is a directory +name. HTTP downloads have a 60-second request timeout and up to three attempts. +Failed asset downloads fail the command before publishing a catalog. Progress +and errors are logged to stderr. -Here's a breakdown of what the script does: - -* The script imports the necessary libraries: `pystac`, `stac_asset`, `asyncio`, `os`, and `sys`. - -* It sets up the configuration for the stac_asset library with `warn=True`. - -* The script defines an `async` function named `main`, which takes a STAC Item URL (href) as an argument. - -* Inside the `main` function: - - * It reads the STAC Item using `pystac.read_file(href)` and stores it in the item variable. - * It creates a directory with the same name as the STAC Item's ID if it doesn't exist using `os.makedirs(item.id, exist_ok=True)`. - * It temporarily changes the current working directory to the newly created directory using `os.chdir(item.id)`. - * It downloads the STAC Item's assets to the current directory using `stac_asset.download_item(item=item, directory=".", config=config)`. - * It returns the STAC Item. - -After defining the `main` function, the script sets the href variable to the first command-line argument provided (`sys.argv[1]`). - -* It uses `asyncio.run` to execute the main function asynchronously with the specified href. - -* In the `main` function, a new STAC Catalog is created: - - * It's given the ID "catalog" and a description and title that mention the staged STAC Item. - * The staged STAC Item is added to the catalog. - * The catalog's hrefs are normalized relative to the current working directory. - * The catalog is saved as a self-contained catalog. +```python linenums="1" title="stage_in_impl.py" +--8<-- +water-bodies/command-line-tools/stage-in/src/stage/stage_in_impl.py +--8<-- +``` ### Step 2 - Create a container for the stage-in diff --git a/docs/cwl-workflow/stage-out.md b/docs/cwl-workflow/stage-out.md index afb34d9..d5b7b96 100644 --- a/docs/cwl-workflow/stage-out.md +++ b/docs/cwl-workflow/stage-out.md @@ -1,59 +1,40 @@ -Below a `stage-out.cwl` CWL (Common Workflow Language) document for a command-line tool that executes a Python script. +The CWL tool invokes the installed `stage-out +` command in `localhost/stage-out:latest`. Existing job inputs and +credential environment variable names are preserved. -The document is designed to use a container and execute a Python script named "stage.py": - -```yaml linenums="1" hl_lines="50-133" +```yaml linenums="1" title="stage-out.cwl" --8<-- cwl-cli/stage-out.cwl --8<-- ``` -The Python script `stage.py` uploads a Spatio-Temporal Asset Catalog (STAC) catalog to an Object Storage S3 bucket. -This script utilizes the `boto3` library for the S3 operations and customizes the STAC I/O to read from and write to an S3 bucket. - -Here's a breakdown of what the script does: - -* The script imports necessary libraries and modules, including `os`, `sys`, `pystac`, `botocore`, `boto3`, `shutil`, and more. - -* It reads command-line arguments to get the STAC catalog URL, S3 bucket name, and a subfolder path to where the catalog will be uploaded. - -* It retrieves the S3 credentials and endpoint URL from environment variables. - -* The script copies the STAC catalog (from the given URL) to a temporary directory under "/tmp/catalog." - -* A custom STAC I/O class (`CustomStacIO`) is defined. This class extends the `DefaultStacIO` class and uses `boto3` to interact with S3. It has methods for writing STAC text content to S3. - -* A connection to S3 is established using `boto3` with the provided S3 credentials and endpoint URL. - -* The default STAC I/O class is set to the custom `CustomStacIO`. - -* The script iterates through items in the STAC catalog, including their assets. For each asset, it uploads the data to the specified S3 bucket under the given subfolder and updates the asset's href to the S3 location. +The package copies the catalog into a unique temporary directory, uploads its +assets, rewrites their links to S3, and publishes the catalog and item documents. +It traverses nested catalogs recursively and leaves the input directory intact. +Upload failures propagate; the command prints the resulting catalog URI to stdout. -* After uploading all assets, it normalizes the catalog's hrefs to point to the S3 location. +Set `aws_access_key_id`, `aws_secret_access_key`, `aws_region_name`, and +`aws_endpoint_url` through the CWL job inputs. The tool passes them as environment +variables to boto3. -* The script then uploads each item in the catalog to S3, updating their hrefs accordingly. - -* Finally, it uploads the catalog itself to S3. - -* The script prints the S3 URL of the uploaded catalog to the standard output. - -This script uploads a STAC catalog and its associated assets to an S3 bucket, making it accessible through the S3 endpoint. - -It uses a custom STAC I/O to facilitate S3 interactions. +```python linenums="1" title="stage_out_impl.py" +--8<-- +water-bodies/command-line-tools/stage-out/src/stage_out/stage_out_impl.py +--8<-- +``` -The AWS credentials and endpoint URL need to be properly configured in the environment variables for the script to work. A +Build the staging images: -```bash linenums="1" hl_lines="8-71" +```bash --8<-- scripts/build-stage-container.sh --8<-- - ``` -To run the stage-out step, one would run: +Run stage-out: -```bash linenums="1" hl_lines="8-71" +```bash --8<-- scripts/cwl-cli-stage-out.sh --8<-- -``` \ No newline at end of file +``` diff --git a/practice-labs/4-CWL-Workflows/2.1-stage-in.ipynb b/practice-labs/4-CWL-Workflows/2.1-stage-in.ipynb index 6555829..7ba51e1 100644 --- a/practice-labs/4-CWL-Workflows/2.1-stage-in.ipynb +++ b/practice-labs/4-CWL-Workflows/2.1-stage-in.ipynb @@ -107,8 +107,10 @@ "podman \\\n", " build \\\n", " --format docker \\\n", - " -t localhost/stage:latest \\\n", - " ${WORKSPACE}/water-bodies/command-line-tools/stage" + " -t localhost/stage-in:latest \\\n", + " ${WORKSPACE}/water-bodies/command-line-tools/stage-in\n", + "\n", + "podman build --format docker -t localhost/stage-out:latest ${WORKSPACE}/water-bodies/command-line-tools/stage-out\n" ] }, { From ffc625f4a3d5d723211a1ee0e21ed8f986769a86 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 20:57:55 +0200 Subject: [PATCH 15/23] fix(labs): align crop examples with packaged commands Pass GeoJSON polygons to crop and separate Podman flags from image names in container help cells. --- practice-labs/1-Application_Steps/crop.ipynb | 4 ++-- practice-labs/2-Containers/crop.ipynb | 8 +++----- practice-labs/2-Containers/normalized-difference.ipynb | 2 +- practice-labs/2-Containers/otsu.ipynb | 2 +- practice-labs/2-Containers/stac.ipynb | 2 +- scripts/crop_green.sh | 2 +- scripts/crop_nir.sh | 2 +- scripts/podman-crop-green.sh | 3 +-- scripts/podman-crop-nir.sh | 3 +-- 9 files changed, 12 insertions(+), 16 deletions(-) diff --git a/practice-labs/1-Application_Steps/crop.ipynb b/practice-labs/1-Application_Steps/crop.ipynb index c61b724..b69c669 100644 --- a/practice-labs/1-Application_Steps/crop.ipynb +++ b/practice-labs/1-Application_Steps/crop.ipynb @@ -167,7 +167,7 @@ "\n", "crop \\\n", " --input-item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --aoi \"-121.399,39.834,-120.74,40.472\" \\\n", + " --aoi='{\"type\":\"Polygon\",\"coordinates\":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \\\n", " --epsg \"4326\" \\\n", " --band green \n" ] @@ -196,7 +196,7 @@ "source": [ "crop \\\n", " --input-item \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A\" \\\n", - " --aoi \"-121.399,39.834,-120.74,40.472\" \\\n", + " --aoi='{\"type\":\"Polygon\",\"coordinates\":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \\\n", " --epsg \"4326\" \\\n", " --band nir " ] diff --git a/practice-labs/2-Containers/crop.ipynb b/practice-labs/2-Containers/crop.ipynb index 0ac0e23..ee144a7 100644 --- a/practice-labs/2-Containers/crop.ipynb +++ b/practice-labs/2-Containers/crop.ipynb @@ -96,7 +96,7 @@ }, "outputs": [], "source": [ - "podman run --rm -itlocalhost/crop:latest crop --help" + "podman run --rm -i localhost/crop:latest crop --help" ] }, { @@ -129,8 +129,7 @@ " --env=HOME=/runs \\\n", " localhost/crop:latest \\\n", " crop \\\n", - " --aoi \\\n", - " \"-121.399,39.834,-120.74,40.472\" \\\n", + " --aoi='{\"type\":\"Polygon\",\"coordinates\":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \\\n", " --band \\\n", " green \\\n", " --epsg \\\n", @@ -167,8 +166,7 @@ " --env=HOME=/runs \\\n", " localhost/crop:latest \\\n", " crop \\\n", - " --aoi \\\n", - " \"-121.399,39.834,-120.74,40.472\" \\\n", + " --aoi='{\"type\":\"Polygon\",\"coordinates\":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \\\n", " --band \\\n", " nir \\\n", " --epsg \\\n", diff --git a/practice-labs/2-Containers/normalized-difference.ipynb b/practice-labs/2-Containers/normalized-difference.ipynb index 0284d62..5647964 100644 --- a/practice-labs/2-Containers/normalized-difference.ipynb +++ b/practice-labs/2-Containers/normalized-difference.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -itlocalhost/norm-diff:latest norm_diff --help" + "podman run --rm -i localhost/norm-diff:latest norm_diff --help" ] }, { diff --git a/practice-labs/2-Containers/otsu.ipynb b/practice-labs/2-Containers/otsu.ipynb index 6678581..9759485 100644 --- a/practice-labs/2-Containers/otsu.ipynb +++ b/practice-labs/2-Containers/otsu.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -itlocalhost/otsu:latest otsu --help" + "podman run --rm -i localhost/otsu:latest otsu --help" ] }, { diff --git a/practice-labs/2-Containers/stac.ipynb b/practice-labs/2-Containers/stac.ipynb index 437340a..41179b2 100644 --- a/practice-labs/2-Containers/stac.ipynb +++ b/practice-labs/2-Containers/stac.ipynb @@ -95,7 +95,7 @@ }, "outputs": [], "source": [ - "podman run --rm -itlocalhost/stac:latest stac --help" + "podman run --rm -i localhost/stac:latest stac --help" ] }, { diff --git a/scripts/crop_green.sh b/scripts/crop_green.sh index 3c85b23..33837b9 100644 --- a/scripts/crop_green.sh +++ b/scripts/crop_green.sh @@ -5,6 +5,6 @@ cd ${RUNTIME} crop \ --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --aoi "-121.399,39.834,-120.74,40.472" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ --epsg "4326" \ --band green \ No newline at end of file diff --git a/scripts/crop_nir.sh b/scripts/crop_nir.sh index 507e9d2..7d652fd 100644 --- a/scripts/crop_nir.sh +++ b/scripts/crop_nir.sh @@ -5,6 +5,6 @@ cd ${RUNTIME} crop \ --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ - --aoi "-121.399,39.834,-120.74,40.472" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ --epsg "4326" \ --band nir \ No newline at end of file diff --git a/scripts/podman-crop-green.sh b/scripts/podman-crop-green.sh index 6c8ad87..cb8ad58 100644 --- a/scripts/podman-crop-green.sh +++ b/scripts/podman-crop-green.sh @@ -9,8 +9,7 @@ podman run \ --env=HOME=/runs \ localhost/crop:latest \ crop \ - --aoi \ - "-121.399,39.834,-120.74,40.472" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ --band \ green \ --epsg \ diff --git a/scripts/podman-crop-nir.sh b/scripts/podman-crop-nir.sh index 897292b..4f9fc18 100644 --- a/scripts/podman-crop-nir.sh +++ b/scripts/podman-crop-nir.sh @@ -9,8 +9,7 @@ podman run \ --env=HOME=/runs \ localhost/crop:latest \ crop \ - --aoi \ - "-121.399,39.834,-120.74,40.472" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ --band \ nir \ --epsg \ From de098fb16c2228d0b14af903f90e31f3711952ae Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:36:27 +0200 Subject: [PATCH 16/23] fix(cwl): allocate sufficient memory for Otsu processing --- cwl-cli/otsu.cwl | 3 ++- cwl-workflow/app-water-bodies-cloud-native.cwl | 3 ++- cwl-workflow/app-water-body-cloud-native.cwl | 3 ++- cwl-workflow/app-water-body.cwl | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cwl-cli/otsu.cwl b/cwl-cli/otsu.cwl index d583dc4..7ff93a4 100644 --- a/cwl-cli/otsu.cwl +++ b/cwl-cli/otsu.cwl @@ -40,7 +40,8 @@ requirements: PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 1024 + ramMax: 1024 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index 3c97254..b5bd774 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -269,7 +269,8 @@ $graph: PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 1024 + ramMax: 1024 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index a0987d5..11afeb5 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -214,7 +214,8 @@ $graph: PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 1024 + ramMax: 1024 NetworkAccess: networkAccess: false DockerRequirement: diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index b9c4ccd..1e669cb 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -211,7 +211,8 @@ $graph: PATH: /app/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ResourceRequirement: coresMax: 1 - ramMax: 512 + ramMin: 1024 + ramMax: 1024 NetworkAccess: networkAccess: false DockerRequirement: From 0303954acdb33b3875f0da8ddcb82cc9d616d25e Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:44:37 +0200 Subject: [PATCH 17/23] feat(dev): add Taskfile and Hatch notebook environments --- .gitignore | 2 + README.md | 6 +- Taskfile.yaml | 115 ++++++++++++++++++ docs/development.md | 58 +++++++++ practice-labs/1-Application_Steps/crop.ipynb | 32 ++--- .../normalized-difference.ipynb | 32 ++--- practice-labs/1-Application_Steps/otsu.ipynb | 32 ++--- practice-labs/1-Application_Steps/stac.ipynb | 37 +++--- scripts/register-hatch-kernel.py | 27 ++++ scripts/requirements-docs.txt | 3 + .../command-line-tools/crop/README.md | 6 + .../command-line-tools/crop/pyproject.toml | 11 +- .../command-line-tools/norm_diff/README.md | 6 + .../norm_diff/pyproject.toml | 11 +- .../command-line-tools/otsu/README.md | 6 + .../command-line-tools/otsu/pyproject.toml | 11 +- .../command-line-tools/stac/README.md | 6 + .../command-line-tools/stac/pyproject.toml | 11 +- .../command-line-tools/stage-in/README.md | 6 + .../stage-in/pyproject.toml | 11 +- .../command-line-tools/stage-out/README.md | 6 + .../stage-out/pyproject.toml | 11 +- 22 files changed, 353 insertions(+), 93 deletions(-) create mode 100644 Taskfile.yaml create mode 100644 docs/development.md create mode 100644 scripts/register-hatch-kernel.py create mode 100644 scripts/requirements-docs.txt diff --git a/.gitignore b/.gitignore index 9b830c4..f407698 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ __pycache__/ *.whl *.puml dist/ +site/ +.venv-fair/ diff --git a/README.md b/README.md index 8fea0cc..5732fbb 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,8 @@ To facilitate this process, participants are guided through the tutorial using i The webpage of the documentation is https://eoap.github.io/mastering-app-package/. -[![License: CC BY-SA 4.0](https://img.shields.io/badge/License-CC_BY--SA_4.0-lightgrey.svg)](https://creativecommons.org/licenses/by-sa/4.0/) \ No newline at end of file +[![License: CC BY-SA 4.0](https://img.shields.io/badge/License-CC_BY--SA_4.0-lightgrey.svg)](https://creativecommons.org/licenses/by-sa/4.0/) +## Developer tooling + +See [developer tooling](docs/development.md) for Task commands, Hatch environments, +and Bash notebook kernels. Run `task --list` from the repository root. diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..a3bb320 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,115 @@ +version: '3' + +vars: + TOOL: crop + WORKFLOW: '{{.ROOT_DIR}}/cwl-workflow/app-water-bodies-cloud-native.cwl' + GENERATED_DIR: '{{.ROOT_DIR}}/runs/generated-cli' + +tasks: + default: + desc: List available developer tasks + cmds: [task --list] + + code:test: + desc: Run one package's tests (TOOL=crop, norm_diff, otsu, stac, stage-in, stage-out) + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:pytest tests {{.CLI_ARGS}}'] + + code:check: + desc: Check Python lint rules without modifying files + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:check {{.CLI_ARGS}}'] + + code:lint: + desc: Check Python formatting without modifying files + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:lint {{.CLI_ARGS}}'] + + code:format: + desc: Format Python source and tests + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:format {{.CLI_ARGS}}'] + + code:typecheck: + desc: Run package type checks + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:typecheck {{.CLI_ARGS}}'] + + code:security: + desc: Run Bandit on package source + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: ['hatch run dev:security {{.CLI_ARGS}}'] + + code:test:all: + desc: Run tests for all six packages + dir: '{{.ROOT_DIR}}' + cmds: + - for package in crop norm_diff otsu stac stage-in stage-out; do task code:test TOOL="$package" || exit; done + - task: code:test:pipeline + + code:test:pipeline: + desc: Test the installed processing commands as a complete chain + dir: '{{.ROOT_DIR}}' + cmds: + - >- + uv run --no-project --with pytest + --with-editable ./water-bodies/command-line-tools/crop + --with-editable ./water-bodies/command-line-tools/norm_diff + --with-editable ./water-bodies/command-line-tools/otsu + --with-editable ./water-bodies/command-line-tools/stac + pytest tests {{.CLI_ARGS}} + + code:test:crop: + desc: Run crop package tests + cmds: [{task: 'code:test', vars: {TOOL: crop}}] + + code:bootstrap: + desc: Generate a CLI preview from CWL into runs/generated-cli (TOOL selects the CWL tool ID) + dir: '{{.ROOT_DIR}}' + cmds: + - >- + uv run --no-project --with 'transpiler-mate-runtime==1.3.0' --with 'cwl2click>=0.10.0' + transpiler-mate cwl2click --clt-id '{{.TOOL}}' --output '{{.GENERATED_DIR}}' '{{.WORKFLOW}}' + + code:bootstrap:crop: + desc: Generate the crop CLI preview + cmds: [{task: 'code:bootstrap', vars: {TOOL: crop}}] + code:bootstrap:norm_diff: + desc: Generate the normalized difference CLI preview + cmds: [{task: 'code:bootstrap', vars: {TOOL: norm_diff}}] + code:bootstrap:otsu: + desc: Generate the Otsu CLI preview + cmds: [{task: 'code:bootstrap', vars: {TOOL: otsu}}] + code:bootstrap:stac: + desc: Generate the STAC CLI preview + cmds: [{task: 'code:bootstrap', vars: {TOOL: stac}}] + + kernel:install: + desc: Register a Hatch-backed Bash kernel (TOOL selects the package) + dir: '{{.ROOT_DIR}}/water-bodies/command-line-tools/{{.TOOL}}' + cmds: + - hatch run default:python '{{.ROOT_DIR}}/scripts/register-hatch-kernel.py' '{{.TOOL}}' {{.CLI_ARGS}} + + kernel:crop: + desc: Register the Bash (crop Hatch) notebook kernel + cmds: [{task: 'kernel:install', vars: {TOOL: crop}}] + + cwl:validate: + desc: Validate every tracked CWL document + dir: '{{.ROOT_DIR}}' + cmds: + - >- + git ls-files -z '*.cwl' | xargs -0 -n 1 + uv run --no-project --with cwltool cwltool --validate + + docs:serve: + desc: Preview the current documentation + dir: '{{.ROOT_DIR}}' + cmds: + - uv run --no-project --with-requirements scripts/requirements-docs.txt mkdocs serve {{.CLI_ARGS}} + + docs:build: + desc: Build the current documentation + dir: '{{.ROOT_DIR}}' + cmds: + - uv run --no-project --with-requirements scripts/requirements-docs.txt mkdocs build {{.CLI_ARGS}} diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..3d9e71c --- /dev/null +++ b/docs/development.md @@ -0,0 +1,58 @@ +# Developer tooling + +Install Python 3.12 or newer, [Task](https://taskfile.dev), Hatch, and uv. +Run tasks from the checkout root. `task --list` shows all commands. + +Select a package with `TOOL=crop`, `norm_diff`, `otsu`, `stac`, `stage-in`, or +`stage-out`. Hatch creates isolated environments and installs the selected package. + +| Command | Purpose | +| --- | --- | +| `task code:test TOOL=crop` | Run one package's tests | +| `task code:test:all` | Run all six package suites and the processing chain | +| `task code:check TOOL=crop` | Check Ruff lint rules | +| `task code:lint TOOL=crop` | Check formatting | +| `task code:format TOOL=crop` | Apply formatting | +| `task code:typecheck TOOL=crop` | Run mypy | +| `task code:security TOOL=crop` | Run Bandit | +| `task cwl:validate` | Validate all tracked CWL documents | +| `task docs:serve` | Preview documentation | +| `task docs:build` | Build documentation | + +Checks report existing diagnostics and return a failing status when they find +issues. `code:format` modifies files; lint checks do not. Pass tool arguments after +`--`, for example `task code:test TOOL=crop -- -q`. + +## Generate a CLI preview + +`task code:bootstrap:crop` runs cwl2click against the scatter workflow and writes +to `runs/generated-cli`. Equivalent aliases exist for `norm_diff`, `otsu`, and +`stac`. Review the preview and reconcile it with the current implementation and +CWL bindings before copying files into a package. Imported named enums currently +generate as string options, so retain the existing CLI choices when reconciling +the preview. Repeated generation replaces +the preview. Override `WORKFLOW` or `GENERATED_DIR` when needed. + +The staging tools use handwritten CLI interfaces. Their CWL documents preserve +existing job inputs; the processing workflow does not contain staging tool IDs. + +## Hatch notebook environments + +Run `task kernel:crop` or `task kernel:install TOOL=otsu` to register a Bash kernel +for a package's default Hatch environment. Select **Bash (crop Hatch)** (or the +corresponding package) in Jupyter or VS Code. The interpreter, PATH, and +VIRTUAL_ENV refer to the Hatch environment, so commands remain available when +the notebook changes directories. Application-step notebooks also show manual +activation for an ordinary Bash kernel. + +Kernel registration writes to the user Jupyter data directory. To install under +a different prefix, use `task kernel:crop -- --prefix /path/to/prefix`. Re-register +a kernel if its Hatch environment is removed or recreated at another location. + +## Documentation preview + +`task docs:serve` uses the same pinned dependencies as the documentation deployment +in `scripts/requirements-docs.txt`. The site groups pages under Home, Build, Run, +Publish, and Reference tabs. Images open in a zoomable viewer with alt-text +captions; Mermaid diagrams retain their normal interaction. Both light and dark +themes use the EOAP palette. `task docs:build` verifies the site locally. diff --git a/practice-labs/1-Application_Steps/crop.ipynb b/practice-labs/1-Application_Steps/crop.ipynb index b69c669..8d599a6 100644 --- a/practice-labs/1-Application_Steps/crop.ipynb +++ b/practice-labs/1-Application_Steps/crop.ipynb @@ -28,21 +28,19 @@ }, "outputs": [], "source": [ - "export WORKSPACE=/workspace/mastering-app-package\n", + "export WORKSPACE=\"$(git rev-parse --show-toplevel)\"\n", "export RUNTIME=${WORKSPACE}/runs\n", - "mkdir -p ${RUNTIME}\n", - "cd ${RUNTIME}" + "mkdir -p \"${RUNTIME}\"\n", + "cd \"${RUNTIME}\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a Python environment\n", + "## Create a Hatch environment\n", "\n", - "Python 3.12 is required for the pinned dependencies. Ensure `python3.12` is installed and available on your PATH before running the following commands.\n", - "\n", - "The Python environment provides a dedicated Python where the `crop` step dependencies are installed:" + "Python 3.12 or newer and Hatch are required. From the repository root, run `task kernel:install TOOL=crop` and select **Bash (crop Hatch)** in the notebook kernel picker. A regular Bash kernel can also activate the Hatch environment with the following cell. Activation keeps the installed command available after changing to the output directory.\n" ] }, { @@ -55,8 +53,13 @@ }, "outputs": [], "source": [ - "python3.12 -m venv ${WORKSPACE}/env_crop\n", - "source ${WORKSPACE}/env_crop/bin/activate\n" + "cd \"${WORKSPACE}/water-bodies/command-line-tools/crop\"\n", + "hatch env create default\n", + "source \"$(hatch env find default)/bin/activate\"\n", + "cd \"${RUNTIME}\"\n", + "\n", + "which python\n", + "which crop\n" ] }, { @@ -89,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install the libraries using pip: " + "Verify the installed package: " ] }, { @@ -102,8 +105,7 @@ }, "outputs": [], "source": [ - "\n", - "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/crop\"" + "crop --help\n" ] }, { @@ -239,7 +241,7 @@ "outputs": [], "source": [ "deactivate\n", - "rm -fr ${WORKSPACE}/env_crop" + "# Keep the Hatch environment for subsequent notebook runs.\n" ] }, { @@ -256,9 +258,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Bash", + "display_name": "Bash (crop Hatch)", "language": "bash", - "name": "bash" + "name": "bash-crop" }, "language_info": { "codemirror_mode": "shell", diff --git a/practice-labs/1-Application_Steps/normalized-difference.ipynb b/practice-labs/1-Application_Steps/normalized-difference.ipynb index 2532b65..9066296 100644 --- a/practice-labs/1-Application_Steps/normalized-difference.ipynb +++ b/practice-labs/1-Application_Steps/normalized-difference.ipynb @@ -28,21 +28,19 @@ }, "outputs": [], "source": [ - "export WORKSPACE=/workspace/mastering-app-package\n", + "export WORKSPACE=\"$(git rev-parse --show-toplevel)\"\n", "export RUNTIME=${WORKSPACE}/runs\n", - "mkdir -p ${RUNTIME}\n", - "cd ${RUNTIME}" + "mkdir -p \"${RUNTIME}\"\n", + "cd \"${RUNTIME}\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a Python environment\n", + "## Create a Hatch environment\n", "\n", - "Python 3.12 is required for the pinned dependencies. Ensure `python3.12` is installed and available on your PATH before running the following commands.\n", - "\n", - "The Python environment provides a dedicated Python where the `norm-diff` step dependencies are installed:" + "Python 3.12 or newer and Hatch are required. From the repository root, run `task kernel:install TOOL=norm_diff` and select **Bash (norm_diff Hatch)** in the notebook kernel picker. A regular Bash kernel can also activate the Hatch environment with the following cell. Activation keeps the installed command available after changing to the output directory.\n" ] }, { @@ -55,8 +53,13 @@ }, "outputs": [], "source": [ - "python3.12 -m venv ${WORKSPACE}/env_norm_diff\n", - "source ${WORKSPACE}/env_norm_diff/bin/activate\n" + "cd \"${WORKSPACE}/water-bodies/command-line-tools/norm_diff\"\n", + "hatch env create default\n", + "source \"$(hatch env find default)/bin/activate\"\n", + "cd \"${RUNTIME}\"\n", + "\n", + "which python\n", + "which norm_diff\n" ] }, { @@ -89,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install the libraries using pip: " + "Verify the installed package: " ] }, { @@ -102,8 +105,7 @@ }, "outputs": [], "source": [ - "\n", - "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/norm_diff\"" + "norm_diff --help\n" ] }, { @@ -196,7 +198,7 @@ "outputs": [], "source": [ "deactivate\n", - "rm -fr ${WORKSPACE}/env_norm_diff" + "# Keep the Hatch environment for subsequent notebook runs.\n" ] }, { @@ -213,9 +215,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Bash", + "display_name": "Bash (norm_diff Hatch)", "language": "bash", - "name": "bash" + "name": "bash-norm_diff" }, "language_info": { "codemirror_mode": "shell", diff --git a/practice-labs/1-Application_Steps/otsu.ipynb b/practice-labs/1-Application_Steps/otsu.ipynb index 393e0f2..d3a6644 100644 --- a/practice-labs/1-Application_Steps/otsu.ipynb +++ b/practice-labs/1-Application_Steps/otsu.ipynb @@ -28,21 +28,19 @@ }, "outputs": [], "source": [ - "export WORKSPACE=/workspace/mastering-app-package\n", + "export WORKSPACE=\"$(git rev-parse --show-toplevel)\"\n", "export RUNTIME=${WORKSPACE}/runs\n", - "mkdir -p ${RUNTIME}\n", - "cd ${RUNTIME}" + "mkdir -p \"${RUNTIME}\"\n", + "cd \"${RUNTIME}\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a Python environment\n", + "## Create a Hatch environment\n", "\n", - "Python 3.12 is required for the pinned dependencies. Ensure `python3.12` is installed and available on your PATH before running the following commands.\n", - "\n", - "The Python environment provides a dedicated Python where the `otsu` step dependencies are installed:" + "Python 3.12 or newer and Hatch are required. From the repository root, run `task kernel:install TOOL=otsu` and select **Bash (otsu Hatch)** in the notebook kernel picker. A regular Bash kernel can also activate the Hatch environment with the following cell. Activation keeps the installed command available after changing to the output directory.\n" ] }, { @@ -55,8 +53,13 @@ }, "outputs": [], "source": [ - "python3.12 -m venv ${WORKSPACE}/env_otsu\n", - "source ${WORKSPACE}/env_otsu/bin/activate\n" + "cd \"${WORKSPACE}/water-bodies/command-line-tools/otsu\"\n", + "hatch env create default\n", + "source \"$(hatch env find default)/bin/activate\"\n", + "cd \"${RUNTIME}\"\n", + "\n", + "which python\n", + "which otsu\n" ] }, { @@ -89,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install the libraries using pip: " + "Verify the installed package: " ] }, { @@ -102,8 +105,7 @@ }, "outputs": [], "source": [ - "\n", - "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/otsu\"" + "otsu --help\n" ] }, { @@ -195,7 +197,7 @@ "outputs": [], "source": [ "deactivate\n", - "rm -fr ${WORKSPACE}/env_otsu" + "# Keep the Hatch environment for subsequent notebook runs.\n" ] }, { @@ -223,9 +225,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Bash", + "display_name": "Bash (otsu Hatch)", "language": "bash", - "name": "bash" + "name": "bash-otsu" }, "language_info": { "codemirror_mode": "shell", diff --git a/practice-labs/1-Application_Steps/stac.ipynb b/practice-labs/1-Application_Steps/stac.ipynb index 56f0b75..9e0ba79 100644 --- a/practice-labs/1-Application_Steps/stac.ipynb +++ b/practice-labs/1-Application_Steps/stac.ipynb @@ -28,21 +28,19 @@ }, "outputs": [], "source": [ - "export WORKSPACE=/workspace/mastering-app-package\n", + "export WORKSPACE=\"$(git rev-parse --show-toplevel)\"\n", "export RUNTIME=${WORKSPACE}/runs\n", - "mkdir -p ${RUNTIME}\n", - "cd ${RUNTIME}" + "mkdir -p \"${RUNTIME}\"\n", + "cd \"${RUNTIME}\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Create a Python environment\n", + "## Create a Hatch environment\n", "\n", - "Python 3.12 is required for the pinned dependencies. Ensure `python3.12` is installed and available on your PATH before running the following commands.\n", - "\n", - "The Python environment provides a dedicated Python where the `stac` step dependencies are installed:" + "Python 3.12 or newer and Hatch are required. From the repository root, run `task kernel:install TOOL=stac` and select **Bash (stac Hatch)** in the notebook kernel picker. A regular Bash kernel can also activate the Hatch environment with the following cell. Activation keeps the installed command available after changing to the output directory.\n" ] }, { @@ -55,8 +53,13 @@ }, "outputs": [], "source": [ - "python3.12 -m venv ${WORKSPACE}/env_stac\n", - "source ${WORKSPACE}/env_stac/bin/activate\n" + "cd \"${WORKSPACE}/water-bodies/command-line-tools/stac\"\n", + "hatch env create default\n", + "source \"$(hatch env find default)/bin/activate\"\n", + "cd \"${RUNTIME}\"\n", + "\n", + "which python\n", + "which stac\n" ] }, { @@ -89,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install the libraries using pip: " + "Verify the installed package: " ] }, { @@ -102,8 +105,7 @@ }, "outputs": [], "source": [ - "\n", - "python -m pip install -e \"${WORKSPACE}/water-bodies/command-line-tools/stac\"" + "stac --help\n" ] }, { @@ -194,7 +196,7 @@ }, "outputs": [], "source": [ - "pip install 'stactools[validate]==0.5.3' requests==2.34.2" + "# Use a separate uv environment for the optional stactools inspector.\n" ] }, { @@ -207,7 +209,8 @@ }, "outputs": [], "source": [ - "stac describe ${RUNTIME}/catalog.json" + "uv run --no-project --with 'stactools[validate]==0.5.3' --with 'requests==2.34.2' \\\n", + " stac describe \"${RUNTIME}/catalog.json\"\n" ] }, { @@ -228,7 +231,7 @@ "outputs": [], "source": [ "deactivate\n", - "rm -fr ${WORKSPACE}/env_stac" + "# Keep the Hatch environment for subsequent notebook runs.\n" ] }, { @@ -245,9 +248,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Bash", + "display_name": "Bash (stac Hatch)", "language": "bash", - "name": "bash" + "name": "bash-stac" }, "language_info": { "codemirror_mode": "shell", diff --git a/scripts/register-hatch-kernel.py b/scripts/register-hatch-kernel.py new file mode 100644 index 0000000..5155a4b --- /dev/null +++ b/scripts/register-hatch-kernel.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Register a Bash kernel using the active Hatch environment's interpreter.""" +import argparse +import json +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from jupyter_client.kernelspec import KernelSpecManager + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('tool', choices=['crop', 'norm_diff', 'otsu', 'stac', 'stage-in', 'stage-out']) +parser.add_argument('--prefix', type=Path, help='Install under this prefix instead of the user Jupyter directory') +args = parser.parse_args() +spec = { + 'argv': [sys.executable, '-m', 'bash_kernel', '-f', '{connection_file}'], + 'display_name': f'Bash ({args.tool} Hatch)', + 'language': 'bash', + 'env': {'PATH': str(Path(sys.executable).parent) + ':${PATH}', 'VIRTUAL_ENV': sys.prefix}, +} +with TemporaryDirectory() as directory: + Path(directory, 'kernel.json').write_text(json.dumps(spec), encoding='utf-8') + installed = KernelSpecManager().install_kernel_spec( + directory, kernel_name='bash-' + args.tool, + user=args.prefix is None, prefix=str(args.prefix) if args.prefix else None, + ) +print(f"Installed {spec['display_name']}: {installed}") diff --git a/scripts/requirements-docs.txt b/scripts/requirements-docs.txt new file mode 100644 index 0000000..97b9fef --- /dev/null +++ b/scripts/requirements-docs.txt @@ -0,0 +1,3 @@ +mkdocs-material==9.7.7 +mkdocs-mermaid2-plugin==1.2.3 +mkdocs-glightbox==0.4.0 diff --git a/water-bodies/command-line-tools/crop/README.md b/water-bodies/command-line-tools/crop/README.md index 141934e..6d2974e 100644 --- a/water-bodies/command-line-tools/crop/README.md +++ b/water-bodies/command-line-tools/crop/README.md @@ -78,3 +78,9 @@ The tests use local STAC items and synthetic rasters; they do not require remote ## CLI source `src/crop/cli.py` defines the console interface. Processing logic lives in `crop_impl.py`. Update the CLI and CWL bindings together when changing arguments. + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=crop` to test this package, +or `task kernel:install TOOL=crop` to register **Bash (crop Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/crop/pyproject.toml b/water-bodies/command-line-tools/crop/pyproject.toml index 454dcc4..4481075 100644 --- a/water-bodies/command-line-tools/crop/pyproject.toml +++ b/water-bodies/command-line-tools/crop/pyproject.toml @@ -103,9 +103,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff] @@ -141,6 +141,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/norm_diff/README.md b/water-bodies/command-line-tools/norm_diff/README.md index a9fe348..7c1a7d5 100644 --- a/water-bodies/command-line-tools/norm_diff/README.md +++ b/water-bodies/command-line-tools/norm_diff/README.md @@ -48,3 +48,9 @@ See [LICENSE.md](LICENSE.md). ## CLI source `src/norm_diff/cli.py` defines the console interface. Processing logic lives in `norm_diff_impl.py`. Update the CLI and CWL bindings together when changing arguments. + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=norm_diff` to test this package, +or `task kernel:install TOOL=norm_diff` to register **Bash (norm_diff Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/norm_diff/pyproject.toml b/water-bodies/command-line-tools/norm_diff/pyproject.toml index f347d61..64f0144 100644 --- a/water-bodies/command-line-tools/norm_diff/pyproject.toml +++ b/water-bodies/command-line-tools/norm_diff/pyproject.toml @@ -101,9 +101,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff] @@ -139,6 +139,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/otsu/README.md b/water-bodies/command-line-tools/otsu/README.md index fbe3123..903926e 100644 --- a/water-bodies/command-line-tools/otsu/README.md +++ b/water-bodies/command-line-tools/otsu/README.md @@ -48,3 +48,9 @@ See [LICENSE.md](LICENSE.md). ## CLI source `src/otsu/cli.py` defines the console interface. Processing logic lives in `otsu_impl.py`. Update the CLI and CWL bindings together when changing arguments. + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=otsu` to test this package, +or `task kernel:install TOOL=otsu` to register **Bash (otsu Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/otsu/pyproject.toml b/water-bodies/command-line-tools/otsu/pyproject.toml index 9e5577a..5d6059b 100644 --- a/water-bodies/command-line-tools/otsu/pyproject.toml +++ b/water-bodies/command-line-tools/otsu/pyproject.toml @@ -102,9 +102,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff] @@ -140,6 +140,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stac/README.md b/water-bodies/command-line-tools/stac/README.md index 2cc53f6..48a8531 100644 --- a/water-bodies/command-line-tools/stac/README.md +++ b/water-bodies/command-line-tools/stac/README.md @@ -48,3 +48,9 @@ See [LICENSE.md](LICENSE.md). ## CLI source `src/stac/cli.py` defines the console interface. Processing logic lives in `stac_impl.py`. Update the CLI and CWL bindings together when changing arguments. + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=stac` to test this package, +or `task kernel:install TOOL=stac` to register **Bash (stac Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/stac/pyproject.toml b/water-bodies/command-line-tools/stac/pyproject.toml index b6a038a..f5f6aba 100644 --- a/water-bodies/command-line-tools/stac/pyproject.toml +++ b/water-bodies/command-line-tools/stac/pyproject.toml @@ -102,9 +102,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff] @@ -140,6 +140,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stage-in/README.md b/water-bodies/command-line-tools/stage-in/README.md index 53036ac..f2ad78a 100644 --- a/water-bodies/command-line-tools/stage-in/README.md +++ b/water-bodies/command-line-tools/stage-in/README.md @@ -49,3 +49,9 @@ See [LICENSE.md](LICENSE.md). Alternatively, install with `python -m pip install -e .` in a Python 3.12 or newer virtual environment. The container runs as UID/GID 2000, with `stage-in` installed in `/app/venv/bin`. Failed asset downloads fail the command before catalog publication. HTTP requests have a 60-second timeout and up to three attempts. + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=stage-in` to test this package, +or `task kernel:install TOOL=stage-in` to register **Bash (stage-in Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/stage-in/pyproject.toml b/water-bodies/command-line-tools/stage-in/pyproject.toml index 2dac65c..c198a4c 100644 --- a/water-bodies/command-line-tools/stage-in/pyproject.toml +++ b/water-bodies/command-line-tools/stage-in/pyproject.toml @@ -103,9 +103,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff.lint] @@ -137,6 +137,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" diff --git a/water-bodies/command-line-tools/stage-out/README.md b/water-bodies/command-line-tools/stage-out/README.md index 919821a..322a81b 100644 --- a/water-bodies/command-line-tools/stage-out/README.md +++ b/water-bodies/command-line-tools/stage-out/README.md @@ -24,3 +24,9 @@ podman build -t localhost/stage-out:latest . and environment settings. Build the image before running that CWL tool. Python 3.12 or newer is required. Alternatively, install with `python -m pip install -e .` in a virtual environment. The multistage image installs a Hatchling-built wheel in `/app/venv` and runs as UID/GID 2000. Licensed under [CC-BY-SA-4.0](LICENSE.md). + +## Developer tasks and notebook kernel + +From the repository root, run `task code:test TOOL=stage-out` to test this package, +or `task kernel:install TOOL=stage-out` to register **Bash (stage-out Hatch)**. +See [developer tooling](../../../docs/development.md) for checks and CLI previews. diff --git a/water-bodies/command-line-tools/stage-out/pyproject.toml b/water-bodies/command-line-tools/stage-out/pyproject.toml index 77bda22..6585201 100644 --- a/water-bodies/command-line-tools/stage-out/pyproject.toml +++ b/water-bodies/command-line-tools/stage-out/pyproject.toml @@ -102,9 +102,9 @@ extra-dependencies = [ skip-install = false dependencies = [ "pytest>=8.4,<10", - "mypy>=2.3.0", - "ruff>=0.16.0", - "bandit>=1.9.4", + "mypy==1.19.1", + "ruff>=0.14,<1", + "bandit>=1.9,<2", ] [tool.ruff.lint] @@ -140,6 +140,7 @@ max-complexity = 4 [tool.hatch.envs.dev.scripts] typecheck = "mypy --install-types --non-interactive {args:src tests}" -lint = "ruff format {args:src tests}" -check = "ruff check --fix {args:src tests}" +lint = "ruff format --check {args:src tests}" +format = "ruff format {args:src tests}" +check = "ruff check {args:src tests}" security = "bandit -r {args:src}" From abada6aa8d0870bb1cc2a05b37c062c5e04e5eea Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:44:37 +0200 Subject: [PATCH 18/23] docs(labs): align guides with packaged tools and current notebooks --- docs/app/water-bodies-detection.md | 5 +- docs/benchmarking/calrissian-benchmark.md | 133 ++++++++++++++- docs/containers/crop.md | 2 +- docs/containers/norm-diff.md | 2 +- docs/containers/otsu.md | 2 +- docs/containers/stac.md | 2 +- docs/cwl-cli/crop.md | 4 +- docs/cwl-cli/norm-diff.md | 2 +- docs/cwl-cli/otsu.md | 2 +- docs/cwl-cli/stac.md | 2 +- docs/cwl-workflow/exec-cloud-native.md | 10 +- docs/cwl-workflow/exec-stage-in.md | 10 +- docs/cwl-workflow/stage-in.md | 2 +- docs/index.md | 28 ++-- docs/kubernetes/calrissian.md | 184 +++++++++++++-------- docs/prerequisites/yaml.md | 2 +- docs/python-env/crop.md | 115 ++++++------- docs/python-env/norm-diff.md | 101 +++++------ docs/python-env/otsu.md | 101 +++++------ docs/python-env/stac.md | 117 +++++++------ docs/scripted/scripted-cloud-native.md | 36 +++- docs/scripted/scripted-k8s-cloud-native.md | 42 ++++- 22 files changed, 596 insertions(+), 308 deletions(-) diff --git a/docs/app/water-bodies-detection.md b/docs/app/water-bodies-detection.md index daa1790..5d571cc 100644 --- a/docs/app/water-bodies-detection.md +++ b/docs/app/water-bodies-detection.md @@ -114,8 +114,9 @@ The crop, normalized difference, Otsu, and STAC steps are Python packages with `pyproject.toml`, a `src/` layout, Hatchling wheel builds, and local tests. Python 3.12 or newer is required. Install a step with `python -m pip install -e water-bodies/command-line-tools/crop` (substitute the -package directory for other steps). The practice notebooks create isolated -virtual environments and install the package instead of separate dependencies. +package directory for other steps). The application-step notebooks use isolated Hatch environments and registered +Bash kernels; see [developer tooling](../development.md) and the linked step +pages for setup. The installed commands are `crop`, `norm_diff`, `otsu`, and `stac`. Use repeated `--rasters` options for the two normalized difference inputs, diff --git a/docs/benchmarking/calrissian-benchmark.md b/docs/benchmarking/calrissian-benchmark.md index ce6ae04..393977e 100644 --- a/docs/benchmarking/calrissian-benchmark.md +++ b/docs/benchmarking/calrissian-benchmark.md @@ -1,8 +1,133 @@ -## Goal +# Benchmark the checked-out workflow -Run the app-water-body-cloud-native.1.0.0.cwl released application package using calrissian, the CWL runner for kubernetes and exploit the usage report. +The lab is `practice-labs/5-Kubernetes/2-benchmark.ipynb`. Complete the +[Calrissian lab](../kubernetes/calrissian.md) first: it prepares +`/calrissian/app-water-bodies-cloud-native.cwl` with local `:metadata` images. +The benchmark runs that packed package at `#main`. -## Lab +Select **Python (Mastering Application Package)** in the deployed editor. +This lab uses its Python environment, including Calrissian, matplotlib, NumPy, +pandas, and PyYAML. The application-step Hatch Bash kernels are for the processing +labs; the benchmark itself is a Python notebook. -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Kubernetes/benchmark.ipynb +## Run the benchmark +The notebook's `Benchmark` class invokes the public Calrissian CLI through +`subprocess.run(..., check=True)`, using `--max-cores 2` and `--max-ram 3G` by +default. Each run creates a separate `/calrissian/benchmark-*` directory: + +| File or directory | Contents | +| --- | --- | +| `params.yaml` | The run's typed workflow inputs | +| `results.json` | Workflow outputs | +| `app.log` | Runner log | +| `usage.json` | Calrissian resource-usage report | +| `logs/` | Individual tool logs | +| `out/` | Final results | +| `tmp*` | Temporary processing outputs | + +An unsuccessful CLI run raises an exception instead of producing a successful +benchmark report. The class and imports are: + + +```python +import random +import json +import subprocess +import tempfile +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import matplotlib.dates as mdates +import yaml + +class Benchmark: + def __init__(self, parameters, max_cores="2", max_ram="3G"): + self.parameters = parameters + self.max_cores = max_cores + self.max_ram = max_ram + self.cwl_path = Path("/calrissian/app-water-bodies-cloud-native.cwl") + + def run(self): + # Use the public CLI and its usage report instead of internal Python APIs. + directory = Path(tempfile.mkdtemp(prefix="benchmark-", dir="/calrissian")) + (directory / "logs").mkdir() + parameters = directory / "params.yaml" + parameters.write_text(yaml.safe_dump(self.parameters)) + subprocess.run([ + "calrissian", "--max-cores", self.max_cores, "--max-ram", self.max_ram, + "--tmp-outdir-prefix", str(directory / "tmp"), + "--outdir", str(directory / "out"), + "--stdout", str(directory / "results.json"), + "--stderr", str(directory / "app.log"), + "--usage-report", str(directory / "usage.json"), + "--tool-logs-basepath", str(directory / "logs"), + "--pod-nodeselectors", "/etc/calrissian/pod-node-selector.yaml", + f"{self.cwl_path}#main", str(parameters), + ], check=True) + return { + "application_package": {"uri": str(self.cwl_path), "cwl": json.loads(self.cwl_path.read_text())}, + "results": json.loads((directory / "results.json").read_text()), + "usage": json.loads((directory / "usage.json").read_text()), + "parameters": self.parameters, + } +``` + +Read the checkout's typed scatter example and execute the first run: + + +```python +parameters = yaml.safe_load(Path("/workspace/mastering-app-package/cwl-workflow/typed-scatter-inputs.yaml").read_text()) +benchmark = Benchmark(parameters) +report = benchmark.run() +``` + +The second run reverses the STAC item order while retaining the other inputs: + + +```python +# Repeat with reversed scatter inputs to compare another run. +parameters = dict(parameters, stac_items=list(reversed(parameters["stac_items"]))) +benchmark = Benchmark(parameters) +``` + + +```python +report = benchmark.run() +``` + +The returned report contains the packed CWL, parameters, workflow results, and +parsed usage report. Both runs retain their files in separate directories. +Changing item order is another execution sample; it does not by itself establish +a performance improvement. + +## Analyze the usage report + +The notebook plots each processing step's start time and elapsed duration as a +Gantt chart. It assigns colors by tool ID from the packed CWL. The tables show +per-step usage and the overall report: + + +```python +usage_df = pd.DataFrame.from_dict(report["usage"]["children"]) + +usage_df +``` + + +```python +import copy + +overall_report = copy.deepcopy(report["usage"]) + +del overall_report["children"] + + +pd.DataFrame.from_dict([overall_report]) +``` + +Run the full notebook to define and call `plot_usage(report)` for the Gantt +chart. The plotted report is the most recent run; retain the first returned +report separately when comparing runs. Actual timings depend on cluster +resources, remote data access, and cache state. diff --git a/docs/containers/crop.md b/docs/containers/crop.md index b657cd3..5b97461 100644 --- a/docs/containers/crop.md +++ b/docs/containers/crop.md @@ -4,7 +4,7 @@ Create a container and run the `crop` step in the container image tagged and bui ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Containers/crop.ipynb +This step has a dedicated lab at `practice-labs/2-Containers/crop.ipynb`. ### Container diff --git a/docs/containers/norm-diff.md b/docs/containers/norm-diff.md index 6d23fb6..52ebd02 100644 --- a/docs/containers/norm-diff.md +++ b/docs/containers/norm-diff.md @@ -4,7 +4,7 @@ Create a container and run the `norm_diff` step in the container image tagged `l ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Containers/norm-diff.ipynb +This step has a dedicated lab at `practice-labs/2-Containers/normalized-difference.ipynb`. ### Container diff --git a/docs/containers/otsu.md b/docs/containers/otsu.md index e8ea2cc..e8ac994 100644 --- a/docs/containers/otsu.md +++ b/docs/containers/otsu.md @@ -4,7 +4,7 @@ Create a container and run the `otsu` step in the container image tagged `localh ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Containers/otsu.ipynb +This step has a dedicated lab at `practice-labs/2-Containers/otsu.ipynb`. ### Container diff --git a/docs/containers/stac.md b/docs/containers/stac.md index f4ad5d0..d5e8429 100644 --- a/docs/containers/stac.md +++ b/docs/containers/stac.md @@ -4,7 +4,7 @@ Create a container and run the `stac` step in the container image tagged `localh ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Containers/stac.ipynb +This step has a dedicated lab at `practice-labs/2-Containers/stac.ipynb`. ### Container diff --git a/docs/cwl-cli/crop.md b/docs/cwl-cli/crop.md index 7674f60..abc7347 100644 --- a/docs/cwl-cli/crop.md +++ b/docs/cwl-cli/crop.md @@ -5,7 +5,7 @@ Wrap the `crop` step as a Common Workflow Language CommandLineTool and execute i ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/3-CWL-CommandLineTools/crop.ipynb +This step has a dedicated lab at `practice-labs/3-CWL-CommandLineTools/crop.ipynb`. ### How to wrap a step as a CWL CommandLineTool @@ -102,4 +102,4 @@ cwltool \ ${WORKSPACE}/cwl-cli/crop-params.yaml ``` -`SchemaDefRequirement` imports the EOAP GeoJSON and URI record schemas. Use the YAML job above to supply structured inputs. The standalone crop tool retains string `epsg` and `band` inputs; the workflows constrain these through enums. The application still crops a rectangle using `bbox`. +`SchemaDefRequirement` imports the EOAP GeoJSON and URI record schemas. Use the YAML job above to supply structured inputs. The standalone crop tool retains string `epsg` and `band` inputs; the workflows constrain these through enums. The binding passes the full Polygon as JSON, and the application masks pixels outside it. diff --git a/docs/cwl-cli/norm-diff.md b/docs/cwl-cli/norm-diff.md index 5abe5ae..13449f6 100644 --- a/docs/cwl-cli/norm-diff.md +++ b/docs/cwl-cli/norm-diff.md @@ -4,7 +4,7 @@ Wrap the `norm_diff` step as a Common Workflow Language CommandLineTool and exec ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/CommandLineTools/norm-diff.ipynb +This step has a dedicated lab at `practice-labs/3-CWL-CommandLineTools/normalized-difference.ipynb`. ### CWL CommandLineTool wrapping the step diff --git a/docs/cwl-cli/otsu.md b/docs/cwl-cli/otsu.md index 77120a0..9abe330 100644 --- a/docs/cwl-cli/otsu.md +++ b/docs/cwl-cli/otsu.md @@ -4,7 +4,7 @@ Wrap the `crop` step as a Common Workflow Language CommandLineTool and exectute ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/CommandLineTools/otsu.ipynb +This step has a dedicated lab at `practice-labs/3-CWL-CommandLineTools/otsu.ipynb`. ### CWL CommandLineTool wrapping the step diff --git a/docs/cwl-cli/stac.md b/docs/cwl-cli/stac.md index e8d2120..5b1fc92 100644 --- a/docs/cwl-cli/stac.md +++ b/docs/cwl-cli/stac.md @@ -4,7 +4,7 @@ Wrap the `stac` step as a Common Workflow Language CommandLineTool and execute i ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/CommandLineTools/stac.ipynb +This step has a dedicated lab at `practice-labs/3-CWL-CommandLineTools/stac.ipynb`. ### CWL CommandLineTool wrapping the step diff --git a/docs/cwl-workflow/exec-cloud-native.md b/docs/cwl-workflow/exec-cloud-native.md index 0ca1a76..0692281 100644 --- a/docs/cwl-workflow/exec-cloud-native.md +++ b/docs/cwl-workflow/exec-cloud-native.md @@ -2,9 +2,15 @@ Run the `app-water-body-cloud-native.1.0.0.cwl` released application package using `cwltool`. -### Lab +### Relationship to the current lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/4-CWL-Workflows/1-cloud-native.ipynb +This page demonstrates a historical released package using the download and +execution scripts below. Its input format and container versions belong to +that release. + +The current notebook, `practice-labs/4-CWL-Workflows/1-cloud-native.ipynb`, +runs the checked-out typed CWL and locally built images instead. Follow the +[current workflow guide](cloud-native.md) for its job format and commands. ### Step 1 - Configure the workspace diff --git a/docs/cwl-workflow/exec-stage-in.md b/docs/cwl-workflow/exec-stage-in.md index 6c1e501..278cfb8 100644 --- a/docs/cwl-workflow/exec-stage-in.md +++ b/docs/cwl-workflow/exec-stage-in.md @@ -2,9 +2,15 @@ Run the `app-water-body.1.0.0.cwl` released application package using `cwltool` on a staged Landsat-9 acquisition. -### Lab +### Relationship to the current lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/4-CWL-Workflows/2.2-staged.ipynb +This page demonstrates a historical released package using the download and +execution scripts below. Its input format and container versions belong to +that release. + +The current notebook, `practice-labs/4-CWL-Workflows/2.2-staged.ipynb`, +runs the checked-out typed CWL and locally built images instead. Follow the +[current workflow guide](staged.md) for its job format and commands. ### Step 1 - Configure the workspace diff --git a/docs/cwl-workflow/stage-in.md b/docs/cwl-workflow/stage-in.md index 40eab7e..4c9e4fa 100644 --- a/docs/cwl-workflow/stage-in.md +++ b/docs/cwl-workflow/stage-in.md @@ -4,7 +4,7 @@ Use a stage-in CWL workflow to stage a Landsat-9 acquisition ### Lab -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Workflows/stage-in.ipynb +This step has a dedicated lab at `practice-labs/4-CWL-Workflows/2.1-stage-in.ipynb`. ### Step 1 - Create a stage-in CWL Workflow diff --git a/docs/index.md b/docs/index.md index 717b755..8066804 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,16 +20,16 @@ This tutorial will guide participants through step-by-step examples, mastering t To facilitate this process, participants are guided through the tutorial using interactive Jupyter Notebooks across various execution scenarios: -1. **Application**: Understand the Application and execute its individual steps (i.e. `crop`, `norm_diff`, `otsu`, `stac`) along with their associated Python modules. +1. **Application**: Understand the Application and execute its individual steps (i.e. `crop`, `norm_diff`, `otsu`, `stac`) using installed console commands in Hatch-managed Python environments. 2. **Containers**: Build container images for each Application step with the required execution software dependencies, then run each step in its respective container. 3. **CWL-CommandLineTool**: Prepare the CWL CommandLineTool document(s) wrapping the command line tool available in their associated container(s), and execute them with `cwltool`. 4. **CWL-Workflow**: Prepare the CWL Workflow orchestrating CWL CommandLineTool document(s) wrapping the command line tool available container(s). Three scenarios are demonstrated: 1. Workflow that reads a single STAC Item as input and orchestrates the four steps `crop`, `norm_diff`, `otsu` and `stac`. 2. Workflow that reads a STAC Catalog of a staged EO acquisition as input, then orchestrates the four steps. A first step to stage-in the EO data is also demonstrated. - 3. Workflow that reads multiple STAC Items as input and scatters them executing the Workflow in scenario 1. + 3. Workflow that reads multiple STAC Items as input and scatters them executing the Workflow in scenario 1. -5. **Kubernetes**: Run the released Application Package CWL using `calrissian`, a CWL runner for kubernetes that creates a pod for each of the Workflow processing step. An additional Notebook is provided to demonstrate the use of the `calrissian` "usage report" feature to generate a resource consumption report of the execution. +5. **Kubernetes**: Pack and run the checked-out Application Package CWL using `calrissian`, a CWL runner for kubernetes that creates a pod for each of the Workflow processing step. An additional Notebook is provided to demonstrate the use of the `calrissian` "usage report" feature to generate a resource consumption report of the execution. ## Overview Flowchart and Key Tools @@ -40,11 +40,11 @@ The tutorial is structured in several parts: timeline title Mastering EO Application Packaging with CWL Part 1 - Water bodies detection - : Short introduction + : Short introduction : Application steps Part 2 - Execution in Python environments - : Create Python environment - : Run Python script + : Create Hatch environment and Bash kernel + : Run installed console commands Part 3 - Package the Application 1/2 : Create and test the containers : Create the CWL CommandLineTool @@ -52,7 +52,7 @@ Part 3 - Package the Application 1/2 Part 4 - Package the Application 2/2 : CWL Workflow for Sentinel-2 Cloud Native processing : CWL Workflow for Landsat-9 processing (includes stage-in/out) - : CWL Workflow of workflows + : CWL Workflow of workflows Part 5 - Release the Application : Continuous Integration : Containers published in a container registry @@ -73,8 +73,8 @@ title Tooling Part 1 - Water bodies detection : N/A Part 2 - Execution in Python environments - : python venv - : python + : Hatch and Task + : Python and Bash notebook kernels Part 3 - Package the Application 1/2 : podman : cwltool @@ -86,5 +86,11 @@ Part 6 - Execution Scenarios : cwltool : calrissian Part 7 - FAIR Application Packages - : N/A -``` \ No newline at end of file + : N/A +``` + +The current Kubernetes labs use locally built images loaded into Minikube with +`:metadata` tags. See [the Kubernetes guide](kubernetes/calrissian.md) for setup. +The [released cloud-native example](cwl-workflow/exec-cloud-native.md) and +[released staged example](cwl-workflow/exec-stage-in.md) document historical +release execution separately from the checkout-based practice notebooks. diff --git a/docs/kubernetes/calrissian.md b/docs/kubernetes/calrissian.md index 99def4a..72504bd 100644 --- a/docs/kubernetes/calrissian.md +++ b/docs/kubernetes/calrissian.md @@ -1,95 +1,149 @@ -### Goal +# Run the checked-out workflow with Calrissian -Run the `app-water-body-cloud-native.1.0.0.cwl` released application package using `calrissian`, a CWL runner for kubernetes. +The lab is `practice-labs/5-Kubernetes/1-calrissian.ipynb`. It packs the current +checkout, runs the typed scatter workflow directly with Calrissian, then submits +the same package as a Kubernetes Job. Calrissian creates a pod for each processing +command-line tool. -`calrissian` creates a pod for each of the Workflow processing step. +## Prepare the local deployment -### Lab +Use the editor and Minikube setup described in +`practice-labs/5-Kubernetes/README.md`. From the +`dev-platform-eoap/mastering-app-package` deployment module directory, the +image-loading task is: -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Kubernetes/calrissian.ipynb +```bash +task labs:kubernetes-images MINIKUBE_PROFILE=eoap-mastering-app-package-docker +``` -### Step 1 - Configure the workspace +This task belongs to the deployment repository, separately from this checkout's +developer Taskfile. Build the four current processing images in the editor first +using the container labs, then load them into the node. Repeat image loading +after a rebuild. The packed package uses `localhost/:metadata` tags so +Kubernetes can use the locally loaded images. -The results produced will be available in the local folder `/workspace/mastering-app-package/runs` +The editor and processing pods need the shared `/calrissian` volume, the +Calrissian service account, and a node-selector configuration. The commands below +use the deployment's `/workspace/mastering-app-package` checkout path. -```bash linenums="1" hl_lines="2-4" title="terminal" ---8<-- -scripts/setup.sh ---8<-- -``` +## Pack the current checkout + +Packing selects `#water-bodies` and renames the selected entrypoint to `#main`. +The notebook then changes local image tags and copies the typed scatter job: -### Step 2 - Download the released Application package ```bash ---8<-- -scripts/download-app-water-bodies-cloud-native.sh ---8<-- +export WORKSPACE=/workspace/mastering-app-package +export RUNTIME=${WORKSPACE}/runs +mkdir -p ${RUNTIME} +cd ${RUNTIME} ``` + +```bash +mkdir -p /calrissian +cwltool --pack "${WORKSPACE}/cwl-workflow/app-water-bodies-cloud-native.cwl#water-bodies" > /calrissian/app-water-bodies-cloud-native.cwl +# A non-latest tag lets Kubernetes use images loaded into the local node. +python - <<'PYCODE' +import json +from pathlib import Path +path = Path("/calrissian/app-water-bodies-cloud-native.cwl") +package = json.loads(path.read_text()) +def use_local_images(value): + if isinstance(value, dict): + for key, child in value.items(): + if key == "dockerPull" and isinstance(child, str) and child.startswith("localhost/") and child.endswith(":latest"): + value[key] = child.removesuffix(":latest") + ":metadata" + else: + use_local_images(child) + elif isinstance(value, list): + for child in value: + use_local_images(child) +use_local_images(package) +path.write_text(json.dumps(package, indent=2)) +PYCODE +cp "${WORKSPACE}/cwl-workflow/typed-scatter-inputs.yaml" /calrissian/params.yaml ``` -sh ${WORKSPACE}/scripts/download-app-water-bodies-cloud-native.sh -``` -### Step 3 - Execute the Application Package +## Execute directly + +`--max-ram` and `--max-cores` limit aggregate resources for running processing +pods. Temporary outputs and final results must be on the shared volume. +`--usage-report` writes resource usage, `--tool-logs-basepath` collects tool logs, +and `--pod-nodeselectors` supplies the deployment's node selectors. + ```bash ---8<-- -scripts/calrissian-cloud-native.sh ---8<-- +mkdir -p /calrissian/logs +calrissian \ + --stdout /calrissian/results.json \ + --stderr /calrissian/app.log \ + --max-ram 3G \ + --max-cores 2 \ + --tmp-outdir-prefix /calrissian/tmp \ + --outdir /calrissian/results \ + --usage-report /calrissian/usage.json \ + --tool-logs-basepath /calrissian/logs \ + --pod-nodeselectors /etc/calrissian/pod-node-selector.yaml \ + /calrissian/app-water-bodies-cloud-native.cwl#main \ + /calrissian/params.yaml ``` -``` -sh ${WORKSPACE}/scripts/calrissian-cloud-native.sh -``` +While the workflow runs, inspect its pods with `kubectl get pods`. +On success, the output JSON points to a STAC catalog directory containing the +item documents and their masks. Inspect it with: -New pods are created: -``` -(base) jovyan@coder-mrossi:~/mastering-app-package$ kubectl get pods -NAME READY STATUS RESTARTS AGE -coder-mrossi 1/1 Running 0 1h -node-crop-2-pod-laqxbhhy 0/1 ContainerCreating 0 8s -node-crop-3-pod-gvdnujzn 0/1 ContainerCreating 0 8s -node-crop-4-pod-qisbfayc 0/1 ContainerCreating 0 8s -node-crop-5-pod-joficzgy 0/1 ContainerCreating 0 8s -node-crop-6-pod-ejarwpxk 0/1 ContainerCreating 0 8s -node-crop-7-pod-ecjgjglq 0/1 ContainerCreating 0 8s -node-crop-pod-ebgkkvpc 0/1 ContainerCreating 0 8s +```bash +tree $( cat /calrissian/results.json | jq -r .stac_catalog.path ) ``` -### Step 4 - Inspect the results +## Submit the Kubernetes Job +The notebook also runs the same package with this manifest: -```bash +```yaml --8<-- -scripts/calrissian-inspect-results.sh +practice-labs/5-Kubernetes/k8s-job.yaml --8<-- ``` -### Expected outcome +The Job uses `eoepca/pde-code-server:amd64` and the installed Calrissian command +at `/opt/calrissian-venv/bin/calrissian`. Adapt the image for a different PDE +build. The manifest mounts `calrissian-claim`, uses `calrissian-sa`, and reads +the packed package and parameters from `/calrissian`. +The notebook copies the node selectors onto that volume, recreates the Job on +repeat runs, and checks both success and failure: + + +```bash +cp /etc/calrissian/pod-node-selector.yaml /calrissian/pod-node-selector.yaml +# Recreate the Job so repeated notebook execution runs it again. +kubectl delete job water-bodies-detection --ignore-not-found +kubectl apply -f "${WORKSPACE}/practice-labs/5-Kubernetes/k8s-job.yaml" +completed=false +for attempt in {1..240}; do + if kubectl wait --for=condition=complete --timeout=5s job/water-bodies-detection 2>/dev/null; then + completed=true + break + fi + if [ "$(kubectl get job water-bodies-detection -o jsonpath='{.status.failed}')" = "1" ]; then + kubectl logs job/water-bodies-detection --tail=100 + break + fi +done +[ "$completed" = true ] ``` -(base) jovyan@coder-mrossi:~/runs$ tree $( cat /calrissian/results.json | jq -r .stac_catalog.path ) -/calrissian/results/tmp3vhr5k8r -├── S2A_10TFK_20210708_0_L2A -│ ├── S2A_10TFK_20210708_0_L2A.json -│ └── otsu.tif -├── S2A_10TFK_20210718_0_L2A -│ ├── S2A_10TFK_20210718_0_L2A.json -│ └── otsu.tif -├── S2A_10TFK_20220504_0_L2A -│ ├── S2A_10TFK_20220504_0_L2A.json -│ └── otsu.tif -├── S2A_10TFK_20220514_0_L2A -│ ├── S2A_10TFK_20220514_0_L2A.json -│ └── otsu.tif -├── S2A_10TFK_20220524_0_L2A -│ ├── S2A_10TFK_20220524_0_L2A.json -│ └── otsu.tif -├── S2B_10TFK_20210713_0_L2A -│ ├── S2B_10TFK_20210713_0_L2A.json -│ └── otsu.tif -└── catalog.json - -6 directories, 13 files -``` \ No newline at end of file + +The Job has no automatic retries (`backoffLimit: 0`) and retains its status for +one hour. On failure, inspect the Job logs and retry the lab after resolving the +cause. Proceed to the [benchmarking lab](../benchmarking/calrissian-benchmark.md) +using the prepared package. + +## Published execution + +The local lab uses the checkout and loaded images. Published deployments need +accessible versioned images and a distributed application package. Historical +release download scripts remain separate examples; they are not the preparation +steps for this lab. diff --git a/docs/prerequisites/yaml.md b/docs/prerequisites/yaml.md index 80a44ff..7ebb74a 100644 --- a/docs/prerequisites/yaml.md +++ b/docs/prerequisites/yaml.md @@ -93,7 +93,7 @@ graph TD In certain circumstances it is necessary to provide multiple values or objects for a single key. -As we've already seen in the [Maps](#Maps) section above, more than one key-value pair can be mapped to a single key. +As we've already seen in the [Maps](#maps) section above, more than one key-value pair can be mapped to a single key. However, it is also possible to define multiple values for a key without having to provide a unique key for each value. diff --git a/docs/python-env/crop.md b/docs/python-env/crop.md index 000add1..78bbdd9 100644 --- a/docs/python-env/crop.md +++ b/docs/python-env/crop.md @@ -1,90 +1,91 @@ ### Goal -Run the `crop` step in a Python virtual environment. +Run `crop` in the same Hatch environment used by the +[practice notebook](https://github.com/eoap/mastering-app-package/blob/feature/metadata/practice-labs/1-Application_Steps/crop.ipynb). +The notebook is stored at `practice-labs/1-Application_Steps/crop.ipynb`. -### Lab +### Requirements -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Application steps/crop.ipynb +Install Python 3.12 or newer, Hatch, and Task. The remote STAC item and its assets must be reachable. +See [developer tooling](../development.md) for the available tasks. -### Step 1 - Configure the workspace +### Register the notebook kernel -The results produced will be available in the local folder `/workspace/mastering-app-package/runs` +From the checkout root, run: -```bash linenums="1" hl_lines="2-4" title="terminal" ---8<-- -scripts/setup.sh ---8<-- +```bash +task kernel:install TOOL=crop ``` -``` -source /workspace/mastering-app-package/scripts/setup.sh -``` +Select **Bash (crop Hatch)** in Jupyter or VS Code. This kernel uses the +package's default Hatch environment. An ordinary Bash kernel can also activate +that environment using the commands below. Run the workspace setup from inside +the checkout so Git can locate its root. -### Step 2 - Create the Python virtual environment +### Configure the workspace -Install the Python package and its pinned dependencies in the virtual environment using `pip`: - -```bash linenums="1" hl_lines="3" title="terminal" ---8<-- -scripts/crop_env.sh ---8<-- +```bash +export WORKSPACE="$(git rev-parse --show-toplevel)" +export RUNTIME=${WORKSPACE}/runs +mkdir -p "${RUNTIME}" +cd "${RUNTIME}" ``` -``` -source ${WORKSPACE}/scripts/crop_env.sh -``` +Outputs are written to `runs/` in the current checkout. -### Step 3 - Crop the green band +### Activate the Hatch environment -The command line tool is invoked to crop the green band: +```bash +cd "${WORKSPACE}/water-bodies/command-line-tools/crop" +hatch env create default +source "$(hatch env find default)/bin/activate" +cd "${RUNTIME}" -```bash linenums="1" hl_lines="7-11" title="terminal" ---8<-- -scripts/crop_green.sh ---8<-- +which python +which crop +crop --help ``` -``` -source ${WORKSPACE}/scripts/crop_green.sh -``` +Hatch installs the Python project and its dependencies; a separate `pip install` +cell is unnecessary. Activation keeps the console command available after changing +to the output directory. -### Step 4 - Crop the NIR band +### Run the step -The command line tool is invoked to crop the NIR band: - -```bash linenums="1" hl_lines="7-11" title="terminal" ---8<-- -scripts/crop_nir.sh ---8<-- +```bash +crop \ + --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ + --epsg "4326" \ + --band green ``` -``` -source ${WORKSPACE}/scripts/crop_nir.sh +```bash +crop \ + --input-item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ + --aoi='{"type":"Polygon","coordinates":[[[-121.399,39.834],[-120.74,39.834],[-120.74,40.472],[-121.399,40.472],[-121.399,39.834]]]}' \ + --epsg "4326" \ + --band nir ``` -### Step 5 - Clean-up +The notebook supplies Polygon JSON. Crop masks pixels outside the polygon, retains the source pixel datatype, and writes LZW-compressed COGs. The command also accepts a comma-separated bounding box. `--epsg 4326` describes the AOI coordinate system. -The Python virtual environment is no longer needed. +### Expected outcome -```bash linenums="1" title="terminal" ---8<-- -scripts/crop_deactivate.sh ---8<-- -``` +The step adds these files under `runs/`: -``` -source ${WORKSPACE}/scripts/crop_deactivate.sh +```text +crop_green.tif +crop_nir.tif ``` -### Expected outcome +Earlier steps' outputs remain available. -The folder `/workspace/mastering-app-package/runs` contains: +### Finish the session +```bash +deactivate ``` -(base) jovyan@coder-mrossi:~/runs$ tree . -. -├── crop_green.tif -└── crop_nir.tif -0 directories, 2 files -``` \ No newline at end of file +Keep the Hatch environment for subsequent runs. The pip-based scripts under +`scripts/` remain an alternative for manual shell use. diff --git a/docs/python-env/norm-diff.md b/docs/python-env/norm-diff.md index f6aeb01..5c3d134 100644 --- a/docs/python-env/norm-diff.md +++ b/docs/python-env/norm-diff.md @@ -1,77 +1,80 @@ ### Goal -Run the `normalized-difference` step in a Python virtual environment. +Run `norm_diff` in the same Hatch environment used by the +[practice notebook](https://github.com/eoap/mastering-app-package/blob/feature/metadata/practice-labs/1-Application_Steps/normalized-difference.ipynb). +The notebook is stored at `practice-labs/1-Application_Steps/normalized-difference.ipynb`. -### Lab +### Requirements -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Application steps/normalized-difference.ipynb +Install Python 3.12 or newer, Hatch, and Task. Run the crop lab first to create `crop_green.tif` and `crop_nir.tif`. +See [developer tooling](../development.md) for the available tasks. -### Step 1 - Configure the workspace +### Register the notebook kernel -The results produced will be available in the local folder `/workspace/mastering-app-package/runs` +From the checkout root, run: -```bash linenums="1" hl_lines="2-4" title="terminal" ---8<-- -scripts/setup.sh ---8<-- +```bash +task kernel:install TOOL=norm_diff ``` -``` -source /workspace/mastering-app-package/scripts/setup.sh -``` +Select **Bash (norm_diff Hatch)** in Jupyter or VS Code. This kernel uses the +package's default Hatch environment. An ordinary Bash kernel can also activate +that environment using the commands below. Run the workspace setup from inside +the checkout so Git can locate its root. -### Step 2 - Create the Python virtual environment +### Configure the workspace -Install the Python package and its pinned dependencies in the virtual environment using `pip`: - -```bash linenums="1" hl_lines="3" title="terminal" ---8<-- -scripts/norm_diff_env.sh ---8<-- +```bash +export WORKSPACE="$(git rev-parse --show-toplevel)" +export RUNTIME=${WORKSPACE}/runs +mkdir -p "${RUNTIME}" +cd "${RUNTIME}" ``` -``` -source ${WORKSPACE}/scripts/norm_diff_env.sh -``` +Outputs are written to `runs/` in the current checkout. -### Step 3 - Calculate the normalized difference +### Activate the Hatch environment -The command line tool is invoked to calculate the normalized difference between the green and NIR cropped bands: +```bash +cd "${WORKSPACE}/water-bodies/command-line-tools/norm_diff" +hatch env create default +source "$(hatch env find default)/bin/activate" +cd "${RUNTIME}" -```bash linenums="1" hl_lines="7-9" title="terminal" ---8<-- -scripts/norm_diff.sh ---8<-- +which python +which norm_diff +norm_diff --help ``` -``` -source ${WORKSPACE}/scripts/norm_diff.sh +Hatch installs the Python project and its dependencies; a separate `pip install` +cell is unnecessary. Activation keeps the console command available after changing +to the output directory. + +### Run the step + +```bash +norm_diff \ + --rasters crop_green.tif \ + --rasters crop_nir.tif ``` -### Step 4 - Clean-up +Supply exactly two rasters with repeated `--rasters` options. Their dimensions, transforms, and coordinate systems must match. -The Python virtual environment is no longer needed. +### Expected outcome -```bash linenums="1" title="terminal" ---8<-- -scripts/norm_diff_deactivate.sh ---8<-- -``` +The step adds these files under `runs/`: -``` -source ${WORKSPACE}/scripts/norm_diff_deactivate.sh +```text +norm_diff.tif ``` -### Expected outcome +Earlier steps' outputs remain available. -The folder `/workspace/mastering-app-package/runs` contains: +### Finish the session +```bash +deactivate ``` -(base) jovyan@coder-mrossi:~/runs$ tree . -. -├── crop_green.tif -├── crop_nir.tif -└── norm_diff.tif - -0 directories, 3 files -``` \ No newline at end of file + +Keep the Hatch environment for subsequent runs. The pip-based scripts under +`scripts/` remain an alternative for manual shell use. diff --git a/docs/python-env/otsu.md b/docs/python-env/otsu.md index d965901..e5fa734 100644 --- a/docs/python-env/otsu.md +++ b/docs/python-env/otsu.md @@ -1,78 +1,79 @@ ### Goal -Run the `otsu` step in a Python virtual environment. +Run `otsu` in the same Hatch environment used by the +[practice notebook](https://github.com/eoap/mastering-app-package/blob/feature/metadata/practice-labs/1-Application_Steps/otsu.ipynb). +The notebook is stored at `practice-labs/1-Application_Steps/otsu.ipynb`. -### Lab +### Requirements -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Application steps/otsu.ipynb +Install Python 3.12 or newer, Hatch, and Task. Run the normalized difference lab first to create `norm_diff.tif`. +See [developer tooling](../development.md) for the available tasks. -### Step 1 - Configure the workspace +### Register the notebook kernel -The results produced will be available in the local folder `/workspace/mastering-app-package/runs` +From the checkout root, run: -```bash linenums="1" hl_lines="2-4" title="terminal" ---8<-- -scripts/setup.sh ---8<-- +```bash +task kernel:install TOOL=otsu ``` -``` -source /workspace/mastering-app-package/scripts/setup.sh -``` +Select **Bash (otsu Hatch)** in Jupyter or VS Code. This kernel uses the +package's default Hatch environment. An ordinary Bash kernel can also activate +that environment using the commands below. Run the workspace setup from inside +the checkout so Git can locate its root. -### Step 2 - Create the Python virtual environment +### Configure the workspace -Install the Python package and its pinned dependencies in the virtual environment using `pip`: - -```bash linenums="1" hl_lines="3" title="terminal" ---8<-- -scripts/otsu_env.sh ---8<-- +```bash +export WORKSPACE="$(git rev-parse --show-toplevel)" +export RUNTIME=${WORKSPACE}/runs +mkdir -p "${RUNTIME}" +cd "${RUNTIME}" ``` -``` -source ${WORKSPACE}/scripts/otsu_env.sh -``` +Outputs are written to `runs/` in the current checkout. -### Step 3 - Calculate the ostu threshold +### Activate the Hatch environment -The command line tool is invoked to calculate Otsu threshold against the normalized difference: +```bash +cd "${WORKSPACE}/water-bodies/command-line-tools/otsu" +hatch env create default +source "$(hatch env find default)/bin/activate" +cd "${RUNTIME}" -```bash linenums="1" hl_lines="7-9" title="terminal" ---8<-- -scripts/otsu.sh ---8<-- +which python +which otsu +otsu --help ``` -``` -source ${WORKSPACE}/scripts/otsu.sh +Hatch installs the Python project and its dependencies; a separate `pip install` +cell is unnecessary. Activation keeps the console command available after changing +to the output directory. + +### Run the step + +```bash +otsu \ + --raster norm_diff.tif ``` -### Step 4 - Clean-up +The output is an unsigned-byte mask containing 0 and 1. Nonfinite input pixels are marked as 0. -The Python virtual environment is no longer needed. +### Expected outcome -```bash linenums="1" title="terminal" ---8<-- -scripts/otsu_deactivate.sh ---8<-- -``` +The step adds these files under `runs/`: -``` -source ${WORKSPACE}/scripts/otsu_deactivate.sh +```text +otsu.tif ``` -### Expected outcome +Earlier steps' outputs remain available. -The folder `/workspace/mastering-app-package/runs` contains: +### Finish the session +```bash +deactivate ``` -(base) jovyan@coder-mrossi:~/runs$ tree . -. -├── crop_green.tif -├── crop_nir.tif -├── norm_diff.tif -└── otsu.tif - -0 directories, 4 files -``` \ No newline at end of file + +Keep the Hatch environment for subsequent runs. The pip-based scripts under +`scripts/` remain an alternative for manual shell use. diff --git a/docs/python-env/stac.md b/docs/python-env/stac.md index b7b841f..e37fede 100644 --- a/docs/python-env/stac.md +++ b/docs/python-env/stac.md @@ -1,82 +1,93 @@ ### Goal -Run the `stac` step in a Python virtual environment. +Run `stac` in the same Hatch environment used by the +[practice notebook](https://github.com/eoap/mastering-app-package/blob/feature/metadata/practice-labs/1-Application_Steps/stac.ipynb). +The notebook is stored at `practice-labs/1-Application_Steps/stac.ipynb`. -### Lab +### Requirements -This step has a dedicated lab available at /workspace/mastering-app-package/practice-labs/Application steps/stac.ipynb +Install Python 3.12 or newer, Hatch, and Task. Run the Otsu lab first to create `otsu.tif`. The source STAC item must be reachable. +See [developer tooling](../development.md) for the available tasks. -### Step 1 - Configure the workspace +### Register the notebook kernel -The results produced will be available in the local folder `/workspace/mastering-app-package/runs` +From the checkout root, run: -```bash linenums="1" hl_lines="2-4" title="terminal" ---8<-- -scripts/setup.sh ---8<-- +```bash +task kernel:install TOOL=stac ``` -``` -source /workspace/mastering-app-package/scripts/setup.sh +Select **Bash (stac Hatch)** in Jupyter or VS Code. This kernel uses the +package's default Hatch environment. An ordinary Bash kernel can also activate +that environment using the commands below. Run the workspace setup from inside +the checkout so Git can locate its root. + +### Configure the workspace + +```bash +export WORKSPACE="$(git rev-parse --show-toplevel)" +export RUNTIME=${WORKSPACE}/runs +mkdir -p "${RUNTIME}" +cd "${RUNTIME}" ``` -### Step 2 - Create the Python virtual environment +Outputs are written to `runs/` in the current checkout. -Install the Python package and its pinned dependencies in the virtual environment using `pip`: +### Activate the Hatch environment -```bash linenums="1" hl_lines="3" title="terminal" ---8<-- -scripts/stac_env.sh ---8<-- -``` +```bash +cd "${WORKSPACE}/water-bodies/command-line-tools/stac" +hatch env create default +source "$(hatch env find default)/bin/activate" +cd "${RUNTIME}" -``` -source ${WORKSPACE}/scripts/stac_env.sh +which python +which stac +stac --help ``` -### Step 3 - Generate the STAC Catalog +Hatch installs the Python project and its dependencies; a separate `pip install` +cell is unnecessary. Activation keeps the console command available after changing +to the output directory. -The command line tool is invoked to produce a STAC Catalog: +### Run the step -```bash linenums="1" hl_lines="7-9" title="terminal" ---8<-- -scripts/stac.sh ---8<-- +```bash +stac \ + --item "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \ + --rasters otsu.tif ``` -``` -source ${WORKSPACE}/scripts/stac.sh -``` +Repeat `--item` and `--rasters` for multiple acquisitions in corresponding order. The catalog copies each mask into its item directory and uses relative asset links. -### Step 4 - Clean-up +### Expected outcome -The Python virtual environment is no longer needed. +The step adds these files under `runs/`: -```bash linenums="1" title="terminal" ---8<-- -scripts/stac_deactivate.sh ---8<-- +```text +catalog.json +S2B_10TFK_20210713_0_L2A/S2B_10TFK_20210713_0_L2A.json +S2B_10TFK_20210713_0_L2A/otsu.tif ``` -``` -source ${WORKSPACE}/scripts/stac_deactivate.sh -``` +Earlier steps' outputs remain available. -### Expected outcome +### Inspect the catalog -The folder `/workspace/mastering-app-package/runs` contains: +The processing package and stactools both provide a command named `stac`. +Run the optional inspector in a separate uv environment so it does not replace +the processing command in the Hatch environment: +```bash +uv run --no-project --with 'stactools[validate]==0.5.3' --with 'requests==2.34.2' \ + stac describe "${RUNTIME}/catalog.json" ``` -(base) jovyan@coder-mrossi:~/runs$ tree . -. -├── S2B_10TFK_20210713_0_L2A -│ ├── S2B_10TFK_20210713_0_L2A.json -│ └── otsu.tif -├── catalog.json -├── crop_green.tif -├── crop_nir.tif -├── norm_diff.tif -└── otsu.tif - -1 directory, 7 files -``` \ No newline at end of file + +### Finish the session + +```bash +deactivate +``` + +Keep the Hatch environment for subsequent runs. The pip-based scripts under +`scripts/` remain an alternative for manual shell use. diff --git a/docs/scripted/scripted-cloud-native.md b/docs/scripted/scripted-cloud-native.md index f87f5c1..22f1841 100644 --- a/docs/scripted/scripted-cloud-native.md +++ b/docs/scripted/scripted-cloud-native.md @@ -1 +1,35 @@ -# TODO \ No newline at end of file +# Script local workflow execution + +Build the processing images first and run this Python example from the checkout +root. It invokes the public cwltool CLI, uses the checked-out typed scatter job, +and reads the workflow's JSON output. Adjust the job's acquisition references +and AOI before running it. + +```python +import json +import subprocess +from pathlib import Path + +workspace = Path.cwd() +output = workspace / "runs" / "scripted" +output.mkdir(parents=True, exist_ok=True) +result = subprocess.run( + [ + "cwltool", "--podman", "--parallel", "--outdir", str(output), + str(workspace / "cwl-workflow/app-water-bodies-cloud-native.cwl") + "#water-bodies", + str(workspace / "cwl-workflow/typed-scatter-inputs.yaml"), + ], + check=True, capture_output=True, text=True, +) +outputs = json.loads(result.stdout) +print(outputs["stac_catalog"]["location"]) +``` + +`check=True` raises an exception on runner failure. Logs are captured in +`result.stderr` on success, or the exception's `stderr` on failure. The output +Directory contains the catalog and referenced masks. + +This example uses the CLI from Python so it can share the same inputs as the +[scatter lab](../cwl-workflow/scatter-cloud-native.md). The historical script +`scripts/cwltool-scripted-execution.py` targets release 1.4.1 and its older +input format; it is separate from this checkout-based example. diff --git a/docs/scripted/scripted-k8s-cloud-native.md b/docs/scripted/scripted-k8s-cloud-native.md index f87f5c1..ae9a8cc 100644 --- a/docs/scripted/scripted-k8s-cloud-native.md +++ b/docs/scripted/scripted-k8s-cloud-native.md @@ -1 +1,41 @@ -# TODO \ No newline at end of file +# Script Calrissian execution + +Complete the [Kubernetes lab](../kubernetes/calrissian.md) first to prepare the +packed workflow, local images, typed parameters, and shared volume. Run this +example in the deployed editor's Python environment: + +```python +import json +import subprocess +import tempfile +from pathlib import Path + +run = Path(tempfile.mkdtemp(prefix="scripted-", dir="/calrissian")) +(run / "logs").mkdir() +subprocess.run( + [ + "calrissian", "--max-cores", "2", "--max-ram", "3G", + "--tmp-outdir-prefix", str(run / "tmp"), + "--outdir", str(run / "out"), + "--stdout", str(run / "results.json"), + "--stderr", str(run / "app.log"), + "--usage-report", str(run / "usage.json"), + "--tool-logs-basepath", str(run / "logs"), + "--pod-nodeselectors", "/etc/calrissian/pod-node-selector.yaml", + "/calrissian/app-water-bodies-cloud-native.cwl#main", + "/calrissian/params.yaml", + ], + check=True, +) +outputs = json.loads((run / "results.json").read_text()) +usage = json.loads((run / "usage.json").read_text()) +print(outputs["stac_catalog"]["location"]) +``` + +Each invocation retains its outputs, logs, and usage report in a separate shared +volume directory. Failure raises an exception. The +[benchmarking notebook](../benchmarking/calrissian-benchmark.md) uses this same +public CLI approach and adds charts and tables. + +The historical `scripts/calrissian-scripted-execution.py` targets older releases +and internal runner APIs; use the CLI example above for the current lab. From 122fa5b1e883edfd0ff28f854361fca133809d16 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:44:45 +0200 Subject: [PATCH 19/23] feat(docs): redesign navigation and add diagrams and image zoom --- .github/workflows/docs.yaml | 8 ++- docs/javascripts/mathjax.js | 4 ++ docs/stylesheets/extra.css | 74 ++++++++++++++++++++++ mkdocs.yml | 118 +++++++++++++++++++++--------------- 4 files changed, 154 insertions(+), 50 deletions(-) create mode 100644 docs/javascripts/mathjax.js create mode 100644 docs/stylesheets/extra.css diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 52b2f10..efd3859 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -10,6 +10,12 @@ on: - 'README.md' - 'docs/**' - 'mkdocs.yml' + - 'scripts/requirements-docs.txt' + - 'water-bodies/command-line-tools/**' + - 'cwl-cli/**' + - 'cwl-workflow/**' + - 'practice-labs/**' + - 'scripts/**' - .github/** jobs: @@ -23,5 +29,5 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.x - - run: pip install mkdocs-material mkdocs-mermaid2-plugin + - run: python -m pip install -r scripts/requirements-docs.txt - run: mkdocs gh-deploy --force \ No newline at end of file diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 0000000..f01b5f8 --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,4 @@ +window.MathJax = { + tex: { inlineMath: [["\\(", "\\)"]], displayMath: [["\\[", "\\]"]] }, + options: { ignoreHtmlClass: ".*|", processHtmlClass: "arithmatex" } +}; diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000..807b2bf --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,74 @@ +/* EOAP training guide: light/dark palette, navigation hierarchy, and tables. */ + +[data-md-color-scheme="default"] { + --md-primary-fg-color: #0f4c5c; + --md-primary-fg-color--light: #17677b; + --md-primary-fg-color--dark: #0a3644; + --md-primary-bg-color: #ffffff; + --md-primary-bg-color--light: rgba(255, 255, 255, .7); + + --md-accent-fg-color: #0d8f8f; + --md-accent-fg-color--transparent: rgba(13, 143, 143, .1); + + --md-typeset-a-color: #0d7c86; +} + +[data-md-color-scheme="slate"] { + + --md-hue: 200; + + --md-primary-fg-color: #10404e; + --md-primary-fg-color--light: #17677b; + --md-primary-fg-color--dark: #0a2c37; + --md-primary-bg-color: rgba(255, 255, 255, .95); + --md-primary-bg-color--light: rgba(255, 255, 255, .7); + + --md-accent-fg-color: #5ee0d0; + --md-accent-fg-color--transparent: rgba(94, 224, 208, .1); + + --md-typeset-a-color: #4fd1c5; +} + +.md-header__button.md-logo :is(img, svg) { + height: 1.6rem; + width: auto; +} + +.md-nav--secondary .md-nav__title, +.md-nav__item--section > .md-nav__link, +.md-nav__item--nested > .md-nav__link { + font-weight: 700; + letter-spacing: .02em; +} + +.md-nav__item--nested .md-nav--secondary, +.md-nav__item--nested > .md-nav { + margin-left: .35rem; + padding-left: .5rem; + border-left: 1px solid var(--md-default-fg-color--lightest); +} + +.md-typeset table:not([class]) th { + background-color: var(--md-primary-fg-color); + color: var(--md-primary-bg-color); + font-weight: 600; +} + +.md-typeset table:not([class]) td code { + white-space: nowrap; +} + +.md-typeset .md-typeset__table { + overflow-x: auto; +} + +.md-typeset .mermaid { + overflow-x: auto; + + text-align: center; +} + +.md-nav__item--nested.md-nav__item--active > .md-nav__container > .md-nav__link, +.md-nav__item--nested.md-nav__item--active > .md-nav__container > .md-nav__link .md-nav__icon { + color: var(--md-typeset-a-color); +} diff --git a/mkdocs.yml b/mkdocs.yml index 44b01ac..80708e6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,6 +3,10 @@ site_name: Mastering Earth Observation Application Packaging with CWL theme: logo: imgs/icon-eoap.png name: material + features: + - navigation.tabs + - navigation.tabs.sticky + - content.code.copy palette: - media: "(prefers-color-scheme: light)" @@ -19,104 +23,120 @@ plugins: - search - mermaid2: version: 10.6.0 + - glightbox: + # The alt text of the image becomes the caption under the enlarged version. + auto_caption: true + caption_position: bottom + # Scroll-to-zoom and drag inside the modal: a full-width UI capture is only + # readable once the reader can get close to a single panel. + zoomable: true + draggable: true + # Never wrap these: a Mermaid diagram is an SVG the reader can already + # read at page size, and the modal would swallow its links. + skip_classes: + - mermaid markdown_extensions: - - pymdownx.details + - toc: + permalink: true - abbr - admonition - footnotes + - tables + - attr_list + - md_in_html + - pymdownx.details - pymdownx.caret - pymdownx.mark - pymdownx.tilde - pymdownx.arithmatex: generic: true - - pymdownx.superfences - - pymdownx.tabbed + - pymdownx.inlinehilite + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + - pymdownx.snippets: + base_path: './' + - pymdownx.tabbed: + alternate_style: true - pymdownx.superfences: custom_fences: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_div_format - - attr_list - - pymdownx.snippets: - base_path: './' - - pymdownx.highlight: - line_spans: __span extra_css: - - styles/css/app.css - + - stylesheets/extra.css extra_javascript: - - javascripts/config.js + - javascripts/mathjax.js - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js - + nav: - - Introduction: 'index.md' - - Prerequisites: - - Introduction to YAML: prerequisites/yaml.md - - Introduction to CWL: prerequisites/cwl.md - - Earth Observation Application Package: - - OGC Activities: app-package/ogc-context.md - - Introducing the application: - - Water bodies detection: app/water-bodies-detection.md - - Application Steps: - - Crop: + - Home: + - Introduction: index.md + - Prerequisites: + - Introduction to YAML: prerequisites/yaml.md + - Introduction to CWL: prerequisites/cwl.md + - Developer tooling: development.md + - EO Application Package overview: app-package/ogc-context.md + - Build an Application: + - Water bodies example: app/water-bodies-detection.md + - Application steps: + - Crop: - Description: app/crop/app.md - Running the step: python-env/crop.md - - Normalized Difference: + - Normalized difference: - Description: app/normalized-difference/app.md - Running the step: python-env/norm-diff.md - Otsu threshold: - Description: app/otsu/app.md - Running the step: python-env/otsu.md - - STAC Catalog: + - STAC catalog: - Description: app/stac/app.md - Running the step: python-env/stac.md - - Package the EO Application: - - Creating the containers: + - Containers: - Scope: containers/scope.md - Build and test the containers: - Crop: containers/crop.md - Normalized difference: containers/norm-diff.md - - Ostu threshold: containers/otsu.md - - STAC Catalog: containers/stac.md - - Create CWL CommandLineTools: + - Otsu threshold: containers/otsu.md + - STAC catalog: containers/stac.md + - CWL CommandLineTools: - Scope: cwl-cli/scope.md - Crop: cwl-cli/crop.md - Normalized difference: cwl-cli/norm-diff.md - - Ostu threshold: cwl-cli/otsu.md - - STAC Catalog: cwl-cli/stac.md - - Create the CWL Workflows: + - Otsu threshold: cwl-cli/otsu.md + - STAC catalog: cwl-cli/stac.md + - CWL workflows: - Scope: cwl-workflow/scope.md - - Sentinel-2 Cloud native Workflow: cwl-workflow/cloud-native.md - - Staged Landsat-9 Workflow: + - Cloud native: cwl-workflow/cloud-native.md + - Staged acquisitions: - Understanding stage-in/out: cwl-workflow/stage-in-out.md - CWL CommandLineTool for stage-in: cwl-workflow/stage-in.md - CWL CommandLineTool for stage-out: cwl-workflow/stage-out.md - Processing a staged Landsat-9 acquisition: cwl-workflow/staged.md - - Sentinel-2 Workflow of Workflow: cwl-workflow/scatter-cloud-native.md - - Release the EO Application: - - Scope: release/scope.md - - Continuous Integration: release/ci.md - - Execution Scenarios: + - Workflow scattering: cwl-workflow/scatter-cloud-native.md + - Run an Application: - Scope: exec/scope.md - - Running on a local machine: + - Historical release examples: - Sentinel-2: cwl-workflow/exec-cloud-native.md - Landsat-9: cwl-workflow/exec-stage-in.md - - Running CWL Workflow on Kubernetes: - - Run the CWL Workflow with calrissian: kubernetes/calrissian.md - - Scripting the execution: - - cwltool API: scripted/scripted-cloud-native.md - - calrissian API: scripted/scripted-k8s-cloud-native.md - - Application benchmarking: benchmarking/calrissian-benchmark.md - - FAIR Application Packages: fair/best-practice.md + - Kubernetes with Calrissian: kubernetes/calrissian.md + - Scripted execution: + - cwltool CLI from Python: scripted/scripted-cloud-native.md + - Calrissian CLI from Python: scripted/scripted-k8s-cloud-native.md + - Benchmarking: benchmarking/calrissian-benchmark.md + - Publish an Application: + - Release process: release/scope.md + - Continuous integration: release/ci.md + - FAIR application packages: fair/best-practice.md - Reference: - Generated CWL reference: reference/cwl/index.md - OGC Application Package Best Practice: reference/ogc-ap-bp.md - CWL CommandLineTool: reference/cwl-commandlinetool.md - CWL Workflow: reference/cwl-workflow.md - - Tools: + - Tools: - cwltool: reference/cwltool.md - - calrissian: reference/calrissian.md + - Calrissian: reference/calrissian.md copyright: License CC BY-SA 4.0, by Creative Commons From a2e87e2a183acbacdbf21dce43bb46129766444b Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:44:46 +0200 Subject: [PATCH 20/23] ci(release): publish verified CWL OCI artifacts with cwl2oci --- .github/workflows/build.yaml | 306 ++++++++++++++---------- .github/workflows/processing-tests.yaml | 7 + docs/release/ci.md | 58 ++++- scripts/check-oci-annotations.py | 30 +++ scripts/prepare-cwl-release.py | 61 +++++ scripts/requirements-cwl-release.txt | 3 + tests/test_release_metadata.py | 46 ++++ 7 files changed, 373 insertions(+), 138 deletions(-) create mode 100644 scripts/check-oci-annotations.py create mode 100644 scripts/prepare-cwl-release.py create mode 100644 scripts/requirements-cwl-release.txt create mode 100644 tests/test_release_metadata.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 52d0ac2..5625cad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,148 +1,204 @@ -name: build +name: Release application packages on: push: - branches: - - master - - main - + branches: [master, main] paths: - # Only rebuild website when apps have changed - - 'water-bodies/**' - - .github/** - - docs/** - - cwl-workflow/** - - scripts/pack-cwl-release.py - - codemeta.json + - 'water-bodies/**' + - '.github/**' + - 'cwl-cli/**' + - 'cwl-workflow/**' + - 'scripts/**' + - 'tests/**' + - 'codemeta.json' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: application-release + cancel-in-progress: false jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.x - - run: pip install cwltool - - run: cwltool --validate cwl-workflow/app-water-bodies-cloud-native.cwl - - run: cwltool --validate cwl-workflow/app-water-body-cloud-native.cwl - - run: cwltool --validate cwl-workflow/app-water-body.cwl + tests: + uses: ./.github/workflows/processing-tests.yaml version: - needs: validate - + needs: tests runs-on: ubuntu-latest - outputs: - app-version: ${{ steps.set-version.outputs.version }} - + version: ${{ steps.metadata.outputs.version }} + registry: ${{ steps.metadata.outputs.registry }} steps: - - uses: actions/checkout@v2 - - run: echo "APP_VERSION=$(cat codemeta.json | jq -r .version )" >> $GITHUB_ENV - - run: echo app version is $APP_VERSION - - id: set-version - run: echo "::set-output name=version::$APP_VERSION" + - uses: actions/checkout@v4 + - id: metadata + name: Check version and publication targets + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + version=$(jq -er '.version' codemeta.json) + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]] + registry="ghcr.io/${GITHUB_REPOSITORY,,}" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "registry=$registry" >> "$GITHUB_OUTPUT" + gh api "repos/$GITHUB_REPOSITORY/releases" --paginate --jq '.[].tag_name' > /tmp/release-tags + gh api "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/$version" --jq '.[].ref' > /tmp/tag-refs + if grep -Fxq "$version" /tmp/release-tags || grep -Fxq "refs/tags/$version" /tmp/tag-refs; then + echo "::error::Version $version already exists. Bump codemeta.json before releasing." + exit 1 + fi container-build: - needs: version - runs-on: ubuntu-latest - + permissions: + contents: read + packages: write strategy: + fail-fast: false matrix: step: [crop, norm_diff, otsu, stac, stage, stage-in, stage-out] - + env: + IMAGE: ${{ needs.version.outputs.registry }}/${{ matrix.step }} + VERSION: ${{ needs.version.outputs.version }} steps: - - uses: actions/checkout@v2 - - run: echo version ${{needs.version.outputs.app-version}} - - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: build & push image - run: | - IMAGE_ID=ghcr.io/eoap/mastering-app-package/${{ matrix.step }} - docker build water-bodies/command-line-tools/${{ matrix.step }} --file water-bodies/command-line-tools/${{ matrix.step }}/Dockerfile --tag ${{ matrix.step }} - docker tag ${{ matrix.step }} $IMAGE_ID:${{needs.version.outputs.app-version}} - docker push $IMAGE_ID:${{needs.version.outputs.app-version}} - - create-release: - - needs: - - container-build - - version - - runs-on: ubuntu-latest - - outputs: - - upload_url: ${{ steps.set-upload-url.outputs.upload_url }} - - steps: - - name: release - uses: actions/create-release@v1 - id: create_release - with: - draft: false - prerelease: false - release_name: ${{needs.version.outputs.app-version}} - tag_name: ${{needs.version.outputs.app-version}} - env: - GITHUB_TOKEN: ${{ github.token }} - - id: set-upload-url - run: echo "::set-output name=upload_url::${{ steps.create_release.outputs.upload_url }}" + - uses: actions/checkout@v4 + - uses: oras-project/setup-oras@v1 + with: + version: '1.3.0' + - name: Authenticate + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + echo "$GH_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Refuse to replace an existing image tag + run: | + oras repo tags "$IMAGE" > /tmp/image-tags 2>/tmp/registry-error || { + # A repository that has never been published has no tags. + grep -Eq 'NAME_UNKNOWN|name unknown|404' /tmp/registry-error || { cat /tmp/registry-error; exit 1; } + } + if grep -Fxq "$VERSION" /tmp/image-tags; then + echo "::error::Image $IMAGE:$VERSION already exists. Use a new release version." + exit 1 + fi + - name: Build image + run: docker build -t "$IMAGE:$VERSION" "water-bodies/command-line-tools/${{ matrix.step }}" + - name: Scan image (block fixable HIGH and CRITICAL vulnerabilities) + uses: aquasecurity/trivy-action@0.35.0 + with: + version: 'v0.69.3' + image-ref: ${{ env.IMAGE }}:${{ env.VERSION }} + scanners: vuln + format: json + output: scan-${{ matrix.step }}.json + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '1' + - uses: actions/upload-artifact@v4 + if: always() + with: + name: scan-${{ matrix.step }} + path: scan-${{ matrix.step }}.json + if-no-files-found: ignore + - name: Push and record immutable image reference + run: | + docker push "$IMAGE:$VERSION" + mkdir -p digests + digest=$(oras resolve "$IMAGE:$VERSION") + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] + printf '%s@%s\n' "$IMAGE" "$digest" > "digests/${{ matrix.step }}.txt" + - uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.step }} + path: digests/*.txt + if-no-files-found: error publish-artifacts: - - needs: - - create-release - - version - + needs: [version, container-build] runs-on: ubuntu-latest - - strategy: - matrix: - step: ["app-water-bodies-cloud-native", "app-water-body-cloud-native", "app-water-body"] - + permissions: + contents: read + packages: write + env: + VERSION: ${{ needs.version.outputs.version }} + REGISTRY: ${{ needs.version.outputs.registry }} steps: - - uses: actions/checkout@v2 - - run: | - for step in crop norm_diff otsu stac - do - tag="ghcr.io/eoap/mastering-app-package/${step}:${{needs.version.outputs.app-version}}" - docker pull ${tag} - shatag=$( docker inspect ${tag} | yq -r '.[0]["RepoDigests"][0]' ) - for cwl in $(ls cwl-workflow/*.cwl) - do - s="${step}" t="${shatag}" yq -i eval '(.$graph[] | select (.id == env(s)) ).requirements.DockerRequirement.dockerPull = env(t)' $cwl + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r scripts/requirements-cwl-release.txt + - uses: oras-project/setup-oras@v1 + with: + version: '1.3.0' + - uses: actions/download-artifact@v4 + with: + pattern: digest-* + merge-multiple: true + path: digests + - name: Prepare and validate packed release CWL + run: python scripts/prepare-cwl-release.py --digests digests --output downloads --version "$VERSION" + - name: Authenticate to GHCR + env: + GH_TOKEN: ${{ github.token }} + run: echo "$GH_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + - name: Annotate, publish, and verify CWL artifacts + shell: bash + run: | + : > downloads/oci-artifacts.txt + for name in app-water-bodies-cloud-native app-water-body-cloud-native app-water-body; do + file="$name.$VERSION.cwl" + annotations="$name.$VERSION.annotations.json" + transpiler-mate cwl2oci \ + --image-source "https://github.com/$GITHUB_REPOSITORY" \ + --image-revision "$GITHUB_SHA" \ + --output "downloads/$annotations" "downloads/$file#main" + python scripts/check-oci-annotations.py "downloads/$annotations" "$file" "$VERSION" + target="$REGISTRY/cwl/$name:$VERSION" + oras repo tags "$REGISTRY/cwl/$name" > /tmp/cwl-tags 2>/tmp/registry-error || { + grep -Eq 'NAME_UNKNOWN|name unknown|404' /tmp/registry-error || { cat /tmp/registry-error; exit 1; } + } + if grep -Fxq "$VERSION" /tmp/cwl-tags; then + echo "::error::Artifact $target already exists. Use a new release version." + exit 1 + fi + (cd downloads && oras push --artifact-type application/cwl \ + --annotation-file "$annotations" "$target" "$file:application/cwl") + digest=$(oras resolve "$target") + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] + reference="$REGISTRY/cwl/$name@$digest" + oras manifest fetch "$reference" > "downloads/$name.$VERSION.manifest.json" + jq -e --slurpfile expected "downloads/$annotations" \ + '.annotations as $actual | $expected[0]."$manifest" | to_entries | all(.[]; $actual[.key] == .value)' \ + "downloads/$name.$VERSION.manifest.json" + mkdir -p "verified/$name" + oras pull "$reference" -o "verified/$name" + cmp "downloads/$file" "verified/$name/$file" + cwltool --validate "verified/$name/$file#main" + printf '%s\n' "$reference" >> downloads/oci-artifacts.txt done - done - - run: | - for cwl in $(ls cwl-workflow/*.cwl) - do - r=$( cat codemeta.json | jq -r ".codeRepository" ) yq -i eval '."s:codeRepository" = {"URL" : env(r)}' $cwl - v="${{needs.version.outputs.app-version}}" yq -i eval '."s:softwareVersion" = env(v)' $cwl + - uses: actions/upload-artifact@v4 + with: + name: application-packages + path: downloads/* + if-no-files-found: error - done - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - run: python -m pip install cwltool - - name: Cleanup downloads folder - run: rm -rf downloads/* - - run: | - mkdir downloads - for cwl in "app-water-bodies-cloud-native" "app-water-body-cloud-native" "app-water-body" - do - python scripts/pack-cwl-release.py cwl-workflow/${cwl}.cwl downloads/${cwl}.${{needs.version.outputs.app-version}}.cwl - done - - uses: actions/upload-artifact@v4 - with: - name: application-package-${{matrix.step}} - path: downloads - overwrite: true - - name: upload linux artifact 1 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ github.token }} - with: - upload_url: ${{needs.create-release.outputs.upload_url}} - asset_path: downloads/${{matrix.step}}.${{needs.version.outputs.app-version}}.cwl - asset_name: ${{matrix.step}}.${{needs.version.outputs.app-version}}.cwl - asset_content_type: text/yaml + create-release: + needs: [version, publish-artifacts] + runs-on: ubuntu-latest + permissions: + contents: write + env: + VERSION: ${{ needs.version.outputs.version }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: application-packages + path: downloads + - name: Create draft, attach verified assets, and publish + run: | + gh release create "$VERSION" --target "$GITHUB_SHA" --title "$VERSION" \ + --draft --notes-file downloads/oci-artifacts.txt + gh release upload "$VERSION" downloads/* + gh release edit "$VERSION" --draft=false diff --git a/.github/workflows/processing-tests.yaml b/.github/workflows/processing-tests.yaml index 4dc8370..2d5f5fe 100644 --- a/.github/workflows/processing-tests.yaml +++ b/.github/workflows/processing-tests.yaml @@ -1,11 +1,15 @@ name: Processing packages on: + workflow_call: pull_request: paths: - 'water-bodies/command-line-tools/**' - 'cwl-cli/**' - 'cwl-workflow/**' - 'tests/**' + - 'scripts/*cwl-release*' + - 'scripts/check-oci-annotations.py' + - '.github/workflows/build.yaml' - '.github/workflows/processing-tests.yaml' push: branches: [main, master] @@ -14,6 +18,9 @@ on: - 'cwl-cli/**' - 'cwl-workflow/**' - 'tests/**' + - 'scripts/*cwl-release*' + - 'scripts/check-oci-annotations.py' + - '.github/workflows/build.yaml' - '.github/workflows/processing-tests.yaml' jobs: test: diff --git a/docs/release/ci.md b/docs/release/ci.md index 21f31f1..fd09460 100644 --- a/docs/release/ci.md +++ b/docs/release/ci.md @@ -30,19 +30,17 @@ This is depicted below: ``` mermaid graph TB -SCM[(software repository)] -SCM -- CWL Workflow --> A -SCM -- codemeta.json --> B -A(validate CWL Workflow) --> B(extract version) -B --> C -subgraph Build containers -SCM -- Dockerfiles --> C -C(build container) --> D(push container) -end -D -- push --> CR[(Container Registry)] -D -- container sha256 --> F("update Dockerpull/metadata in CWL Workflows") -F -- push --> AR[(Artifact Registry)] -SCM -- codemeta.json --> F +SCM[(software repository)] --> T(Test packages and validate CWL) +T --> V(Check new release version) +V --> B(Build images) +B --> S(Scan images) +S --> CR[(Push images to GHCR)] +CR --> P(Update metadata and image digests) +P --> C(Pack and validate CWL) +C --> A(Generate annotations with cwl2oci) +A --> O[(Publish CWL with ORAS)] +O --> R(Pull by digest and verify) +R --> G(Publish GitHub release and assets) ``` Below an example of a GitHub CI configuration implementing the scenario: @@ -59,3 +57,37 @@ Release image digests are written to mandatory `requirements.DockerRequirement` entries. `scripts/pack-cwl-release.py` selects the workflow explicitly, embeds the schema imports, preserves package metadata, and validates the downloaded artifact as a standalone CWL file. + +The release workflow calls the processing test workflow before publishing anything. +Set a new version in `codemeta.json`; an existing GitHub release or tag is rejected. +The workflow runs on changes to release inputs on `main` or `master`, or manually +with `workflow_dispatch`. Concurrent releases are serialized. + +Each image is built and scanned with Trivy before it is pushed. Fixable HIGH and +CRITICAL vulnerabilities block publication; the JSON report is retained as a run +artifact, including on scan failure. Unfixed vulnerabilities do not block this +gate. The image jobs record immutable GHCR references for the packaging job, +which updates mandatory Docker requirements and release metadata in isolated +copies of the workflows. Repository source files are not rewritten. + +After packing and validation, pinned `transpiler-mate-runtime` and `cwl2oci` +generate the OCI annotations from each final CWL document's `#main` entrypoint. +The generated `$manifest` structure is passed directly to ORAS. The entrypoint +annotation is normalized from the runtime's process identifier to the artifact filename +plus `#main`; source and revision identify the repository and release commit. +These annotations describe the CWL artifacts; they are not Docker image labels. + +ORAS publishes the three workflows at +`ghcr.io///cwl/:`, using the project's +`application/cwl` artifact and layer media type. Existing image and CWL tags are +rejected rather than replaced. Each artifact is pulled by digest, compared with +the packed file, checked against the generated annotations, and validated again. `oci-artifacts.txt` records immutable +references. CWL files, annotation JSON files, OCI manifests, and that inventory are attached to +a GitHub draft release, which is published only after every verification and +asset upload succeeds. A failed upload leaves a draft for inspection. + +The workflow uses `GITHUB_TOKEN` with `packages: write` only for registry jobs +and `contents: write` only for release creation. GHCR must allow the repository's +token to publish these packages. A failure after image or OCI publication can +leave registry artifacts without a GitHub release; inspect the failed run and +use a new version for the next release. Registry publication is not transactional. diff --git a/scripts/check-oci-annotations.py b/scripts/check-oci-annotations.py new file mode 100644 index 0000000..b62c666 --- /dev/null +++ b/scripts/check-oci-annotations.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Check cwl2oci output and make the entrypoint portable within the OCI artifact.""" +import json +import sys +from pathlib import Path +from urllib.parse import urlsplit + +path = Path(sys.argv[1]) +filename, version = sys.argv[2:] +document = json.loads(path.read_text()) +annotations = document['$manifest'] +required = ( + 'org.opencontainers.image.title', 'org.opencontainers.image.description', + 'org.opencontainers.image.version', 'org.opencontainers.image.licenses', + 'org.opencontainers.image.source', 'org.opencontainers.image.revision', + 'org.cwl.entrypoint', 'org.cwl.spec', 'org.cwl.type', +) +for key in required: + if not isinstance(annotations.get(key), str) or not annotations[key]: + raise ValueError(f'Missing or invalid annotation: {key}') +if annotations['org.opencontainers.image.version'] != version: + raise ValueError('OCI version does not match release version') +entrypoint = annotations['org.cwl.entrypoint'] +if entrypoint != 'main' and urlsplit(entrypoint).fragment != 'main': + raise ValueError('Expected packed workflow entrypoint #main') +if annotations['org.cwl.type'] != 'Workflow': + raise ValueError('Expected a Workflow artifact') +# Runtime versions return either 'main' or a resolved URI. Use the artifact filename. +annotations['org.cwl.entrypoint'] = f'{filename}#main' +path.write_text(json.dumps(document, indent=2) + '\n') diff --git a/scripts/prepare-cwl-release.py b/scripts/prepare-cwl-release.py new file mode 100644 index 0000000..6dd3035 --- /dev/null +++ b/scripts/prepare-cwl-release.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Prepare isolated release workflows using digests emitted by the image jobs.""" +import argparse +import json +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +from ruamel.yaml import YAML + +WORKFLOWS = ( + 'app-water-bodies-cloud-native', + 'app-water-body-cloud-native', + 'app-water-body', +) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--digests', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--version', required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='cwl-release-') as staging_directory: + prepared = Path(staging_directory) + shutil.copytree('cwl-workflow', prepared, dirs_exist_ok=True) + yaml = YAML() + repository = json.loads(Path('codemeta.json').read_text())['codeRepository'] + for name in WORKFLOWS: + source = Path('cwl-workflow') / f'{name}.cwl' + document = yaml.load(source.read_text()) + for process in document['$graph']: + requirements = process.get('requirements', {}) + if isinstance(requirements, list): + docker = next((r for r in requirements if r['class'] == 'DockerRequirement'), None) + else: + docker = requirements.get('DockerRequirement') + if docker is None: + continue + tool = process['id'].removeprefix('#') + reference = (args.digests / f'{tool}.txt').read_text().strip() + if not re.fullmatch(r'ghcr\.io/[a-z0-9_./-]+@sha256:[a-f0-9]{64}', reference): + raise ValueError(f'Invalid immutable image reference for {tool}: {reference}') + docker['dockerPull'] = reference + document['s:softwareVersion'] = args.version + document['s:codeRepository'] = repository + target = prepared / source.name + with target.open('w') as stream: + yaml.dump(document, stream) + subprocess.run([ + sys.executable, 'scripts/pack-cwl-release.py', str(target), + str(args.output / f'{name}.{args.version}.cwl'), + ], check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/requirements-cwl-release.txt b/scripts/requirements-cwl-release.txt new file mode 100644 index 0000000..cc6ae64 --- /dev/null +++ b/scripts/requirements-cwl-release.txt @@ -0,0 +1,3 @@ +transpiler-mate-runtime==1.3.0 +cwl2oci==0.1.1 +cwltool==3.2.20260720092025 diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 0000000..b389940 --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,46 @@ +"""Release metadata must remain portable and match the published version.""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +CHECKER = Path(__file__).resolve().parents[1] / 'scripts/check-oci-annotations.py' + + +@pytest.mark.parametrize('entrypoint', ['main', '#main', 'file:///runner/work/release.cwl#main']) +def test_annotation_entrypoint_is_portable(tmp_path, entrypoint): + annotations = {key: 'value' for key in ( + 'org.opencontainers.image.title', 'org.opencontainers.image.description', + 'org.opencontainers.image.licenses', 'org.opencontainers.image.source', + 'org.opencontainers.image.revision', 'org.cwl.spec', + )} + annotations.update({ + 'org.opencontainers.image.version': '2.0.0', + 'org.cwl.entrypoint': entrypoint, + 'org.cwl.type': 'Workflow', + }) + path = tmp_path / 'annotations.json' + path.write_text(json.dumps({'$manifest': annotations})) + subprocess.run([sys.executable, str(CHECKER), str(path), 'workflow.cwl', '2.0.0'], check=True) + actual = json.loads(path.read_text())['$manifest'] + assert actual == {**annotations, 'org.cwl.entrypoint': 'workflow.cwl#main'} + # A version mismatch must prevent publishing the artifact. + result = subprocess.run( + [sys.executable, str(CHECKER), str(path), 'workflow.cwl', '3.0.0'], + capture_output=True, text=True, + ) + assert result.returncode != 0 + assert 'version does not match' in result.stderr + + +def test_incomplete_annotations_are_rejected(tmp_path): + path = tmp_path / 'annotations.json' + path.write_text(json.dumps({'$manifest': {'org.cwl.entrypoint': 'main'}})) + result = subprocess.run( + [sys.executable, str(CHECKER), str(path), 'workflow.cwl', '2.0.0'], + capture_output=True, text=True, + ) + assert result.returncode != 0 + assert 'Missing or invalid annotation' in result.stderr From 3acd20902f39f2d732008cdd31966c7f03a15d31 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:44:46 +0200 Subject: [PATCH 21/23] docs(fair): add citation and RO-Crate publication exercises --- codemeta.json | 2 +- docs/fair/best-practice.md | 300 +++++++++++++++++++++--------- scripts/requirements-cwl-fair.txt | 4 + 3 files changed, 212 insertions(+), 94 deletions(-) create mode 100644 scripts/requirements-cwl-fair.txt diff --git a/codemeta.json b/codemeta.json index c3c4312..2663ca6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -1,7 +1,7 @@ { "@context": "https://doi.org/10.5063/schema/codemeta-2.0", "@type": "SoftwareSourceCode", - "license": "https://spdx.org/licenses/CC-BY-NC-SA-4.0", + "license": "https://spdx.org/licenses/CC-BY-SA-4.0", "codeRepository": "https://github.com/eoap/mastering-app-package.git", "dateCreated": "2022-09-01", "datePublished": "2022-09-25", diff --git a/docs/fair/best-practice.md b/docs/fair/best-practice.md index 6bd4a35..42dbfc9 100644 --- a/docs/fair/best-practice.md +++ b/docs/fair/best-practice.md @@ -1,127 +1,241 @@ -## Application Package Software Configuration Management +# Publish a FAIR application package -The SCM has the task of tracking and controlling changes in the software as a part of the larger cross-disciplinary field of configuration management. +A reusable application package needs a clear description, identifiable software +and containers, instructions for running it, and a way to credit its authors. +This tutorial turns the water-bodies workflow into a citable research object +using [Transpiler-Mate](https://github.com/transpiler-mate) plugins. -SCM practices include revision control and the establishment of baselines. +The exercise generates files locally. The final, optional exercise publishes a +record to **Zenodo Sandbox**. Keep sandbox identifiers separate from production +release metadata. -The Application Package code is hosted on a repository publicly accessible (Github, Bitbucket, a GitLab instance, an institutional software forge, etc.) using one of the version control systems supported by (Subversion, Mercurial and Git) +## What you will produce -The Application Package code include, at the top level of the source code tree, the following files: +| Result | Purpose | Generator | +| --- | --- | --- | +| Packed CWL | Executable workflow with embedded tool and schema definitions | Release CI / cwltool | +| OCI annotations and immutable registry reference | Describe and retrieve a specific workflow artifact | cwl2oci / ORAS | +| CFF, BibTeX, RIS and CSL-JSON citations | Credit the workflow's authors and identify its version | cwl2citation | +| Workflow RO-Crate ZIP | Package the workflow, metadata and supporting files | cwl2rocrate | +| Sandbox record and DOI | Practise depositing and identifying the package | invenio-publish | -* README containing a description of the software (name, purpose, pointers to website, documentation, development platform, contact, and support information, …) -* AUTHORS, a list of all the persons to be credited for the software. -* LICENSE, the project license terms. For Open Source Licenses, the standard SPDX license names are used. For large software projects and developers, the REUSE (https://reuse.software/) process and tools can be an option to look at. -* codemeta.json, a linked data metadata file that helps index the source code in the Software Heritage archive and provides an easy way to link to other related research outputs. +These outputs support FAIR practices; generating them does not automatically +make the workflow discoverable in a catalog or preserve every dependency. -The code meta project motivation (https://codemeta.github.io/) is reported below: +## Prepare the environment -**Research relies heavily on scientific software, and a large and growing fraction of researchers are engaged in developing software as part of their own research (Hannay et al 2009). Despite this, infrastructure to support the preservation, discovery, reuse, and attribution of software lags substantially behind that of other research products such as journal articles and research data. This lag is driven not so much by a lack of technology as it is by a lack of unity: existing mechanisms to archive, document, index, share, discover, and cite software contributions are heterogeneous among both disciplines and archives and rarely meet best practices (Howison 2015). Fortunately, a rapidly growing movement to improve preservation, discovery, reuse, and attribution of academic software is now underway: a recent NIH report; conferences and working groups of FORCE11, WSSSPE & Software Sustainability Institute; and the rising adoption of repositories like GitHub, Zenodo, figshare & DataONE by academic software developers. Now is the time to improve how these resources can communicate to each other.** +Run from the repository root, with Python 3.12 available: -CodeMeta developed the translations from the different vocabularies. The CodeMeta vocabulary is an extension of the SoftwareApplication and SoftwareSourceCode classes found in the vocabulary of the Schema.org initiative [schema]. Metadata information conformant to the CodeMeta vocabulary can be represented in JSON format, named codemeta.json, like the example below with the information for Water Bodies Detection application. - -```json linenums="1" title="codemeta.json" ---8<-- -codemeta.json ---8<-- +```bash +python3.12 -m venv .venv-fair +source .venv-fair/bin/activate +python -m pip install -r scripts/requirements-cwl-fair.txt +mkdir -p runs/fair ``` -## Application Package Software identification and traceability - -In a Reproducible FAIR Workflow scenario, ensuring that the code behind the application is uniquely identified and traceable is necessary. Even if software configuration management systems have become a common tool for tracking and controlling changes in the software, the FAIR publication of applications also needs methods for software citation, software retrieval, and long-term preservation. This is achieved with the issuing of a persistent identifier. +The pinned plugins share the `transpiler-mate` executable. The RO-Crate plugin's +package name is `cwl2ro-crate`, but its command is `cwl2rocrate`. -The persistent identifier provides a long-lasting reference to source code and is usually understood as an actionable and accessible reference over the Internet that retrieves the necessary files. As such, the persistent component is dependent on the service commitment to resolve the identifier and the dedicated storage lifespan. +## Check the metadata before generating artifacts -Several initiatives provide this persistent dual functionality of identification and archivation. This Best Practice identifies one solution with Software Heritage. Software Heritage is a non-profit multi-stakeholder initiative unveiled in 2016 by the French Institute for Research in Computer Science and Automation (Inria) and supported by UNESCO. The mission of Software Heritage is to collect, preserve, and share all software that is publicly available in source code form, with the goal of building a common shared infrastructure at the service of industry, research, culture, and society. +The workflow's document-level `s:` fields describe the application. Its selected +process defines the executable interface. Inspect both: -The __Software Heritage__ provides SoftWare Heritage persistent IDentifiers (SWHIDs) and ensures preservation of the software source code by crawling code hosting platforms, like GitHub, GitLab.com, or Bitbucket, and package archives, like npm or PyPI, ingested into a special data structure, a Merkle DAG, that is the core of the archive. +```bash +sed -n '1,38p' cwl-workflow/app-water-bodies-cloud-native.cwl +cat codemeta.json +``` -To obtain a SWHID, the code must be hosted on a publicly accessible repository (Github, Bitbucket, a GitLab instance, an institutional software forge, etc.) using one of the version control systems supported by Software Heritage (Subversion, Mercurial, and Git). +Check the following before a real publication: + +- Replace Jane Doe and John Doe training examples with the actual authors and + affiliations. Preserve their intended order in citations. +- Match the license in `LICENSE.md`, CodeMeta and CWL. This repository uses + `CC-BY-SA-4.0`; the SPDX URL in CodeMeta and the CWL license identifier must agree. +- Give the package a meaningful name, description, source repository and help URL. +- Distinguish creation dates from release dates. Do not present `dateCreated` + as the publication date of a new version. +- Use a new release version and check that the generated artifacts describe it. + +`codemeta.json` currently supplies the release version to CI. Source workflows +can retain their individual development versions; the release preparation job +sets the version and repository on copies before packing. Generate release +citations and crates from those final copies, so all published outputs agree. +The current `1.1.0` CodeMeta version is already released and must be bumped for +a new publication through CI. + +The [release tutorial](../release/ci.md) explains that process. Keep generated +CodeMeta separate from the CI input: the +[cwl2codemeta plugin](https://github.com/transpiler-mate/cwl2codemeta) converts CWL +metadata to CodeMeta 3.0, while this repository's root file uses CodeMeta 2.0. +Switching the authoritative metadata source requires an explicit migration. + +## Select and pack the workflow + +For a local practice run, pack the checked-out scattering workflow: + +```bash +python scripts/pack-cwl-release.py \ + cwl-workflow/app-water-bodies-cloud-native.cwl \ + runs/fair/workflow.cwl +export FAIR_WORKFLOW="$(pwd)/runs/fair/workflow.cwl" +cwltool --validate "${FAIR_WORKFLOW}#main" +``` -## Application Package Containers identification +The source entrypoint is `#water-bodies`; the packing script selects it and +renames it to `#main`. It also preserves package metadata and embeds imported +schema definitions. -A Reproducible FAIR Workflow scenario also requires that the application container(s) is(are) uniquely identified and traceable. +For a release exercise, substitute the verified packed CWL downloaded from +GitHub or pulled from GHCR for `FAIR_WORKFLOW`. The local practice file still +contains development image references. Release files use immutable image +references recorded by CI. -Using the sha256 signature of the container, containers have an identifier. +A container or workflow reference ending in `@sha256:...` identifies registry +content. Retention and access still depend on the registry. A source commit, +a workflow artifact digest, a Software Heritage identifier and a DOI identify +different objects; record the relationships between them. -Example: +## Generate citations -```yaml -- class: CommandLineTool - id: crop +Generate citations from the selected workflow's document-level metadata: - hints: - DockerRequirement: - dockerPull: docker.io/terradue/crop@sha256:ec2d8e71ab5834cb9db01c5001bde9c3d6038d0418ad085726b051b4359750e1 +```bash +transpiler-mate cwl2citation \ + --code-repository https://github.com/eoap/mastering-app-package.git \ + --output runs/fair/citations \ + "${FAIR_WORKFLOW}#main" ``` -## Application Package metadata +Inspect `CITATION.cff`, `citation.bib`, `citation.ris`, `citation.csl.json` and +`citation.txt` under `runs/fair/citations`. Check the authors, title and version. +CFF and CSL-JSON are schema-validated by the plugin. An absent DOI remains absent; +this command does not register one. -The OGC Best Practices for EO Application Packages (OGC 20-089) recommends that to enrich the application package with new concepts, these should originate from schema.org and be linked with their RDF encoding. While OGC 20-089 only enforces the version element as a mandatory, it already suggest several additional elements that are key for the Reproducible FAIR Workflows scenario such as the following. +The plugin rejects existing output files. Use a fresh destination for another +run. For a real repository release, review the generated `CITATION.cff` before +placing it at the repository root; this local exercise leaves it under `runs/`. +See the [citation plugin documentation](https://github.com/transpiler-mate/cwl2citation) +for release dates, citation styles and individual output formats. -* **author**: The main author of the Application Package - https://schema.org/author. -* **citation**: A citation or reference to a publication, web page, scholarly article, etc. https://schema.org/citation. -* **codeRepository**: Link to the repository where the Application code is located (e.g., SVN, github). https://schema.org/codeRepository. -* **contributor**: A secondary contributor to the Application Package https://schema.org/contributor. -* **dateCreated**: The date on which the Application Package was created. https://schema.org/dateCreated. -* **keywords**: Keywords used to describe this application. Multiple entries in a keywords list are delimited by commas. https://schema.org/keywords. -* **license**: An URL to the license document that applies to this application. https://schema.org/license. -* **releaseNotes**: Description of what changed in this version. https://schema.org/releaseNotes. +## Package a Workflow RO-Crate -All these elements can be obtained from the CodeMeta vocabulary (i.e., codemeta.json) and can be directly added when defining the Application Package (e.g. during the Continuous Integration). +Include the generated citation and a runnable input example with the workflow: -Example: +```bash +transpiler-mate cwl2rocrate \ + --output runs/fair/workflow-crate \ + --zip \ + --attach runs/fair/citations/CITATION.cff \ + --attach cwl-workflow/typed-cloud-native-inputs.yaml \ + "${FAIR_WORKFLOW}#main" +``` +Inspect `runs/fair/workflow-crate/ro-crate-metadata.json`, its packed +`workflow.cwl`, and the files under `attachments/`. The ZIP is written beside +the directory as `runs/fair/workflow-crate.zip`. + +The plugin validates the crate against the bundled REQUIRED-level profiles. +Validation can need network access for JSON-LD contexts. Use a new output +location when repeating the exercise. + +A Workflow RO-Crate describes and packages the workflow. It does not execute it +or download referenced container images and external STAC assets. The input +example can reference external data whose availability must be considered +separately. See the [RO-Crate plugin documentation](https://github.com/transpiler-mate/cwl2ro-crate) +for attachment and validation details. + +## Optional: publish to Zenodo Sandbox + +This step creates and **publishes** a sandbox record. Review the workflow's +metadata and attachments first. Obtain a token from your own +[Zenodo Sandbox](https://sandbox.zenodo.org/) account; production Zenodo tokens +and records are separate. + +Read the token without putting it into shell history: + +```bash +read -rsp 'Zenodo Sandbox token: ' INVENIO_TOKEN +printf '\n' +export INVENIO_TOKEN +transpiler-mate invenio-publish \ + --base-url https://sandbox.zenodo.org/ \ + --auth-token "$INVENIO_TOKEN" \ + --attach "$FAIR_WORKFLOW" \ + --attach runs/fair/workflow-crate.zip \ + --attach runs/fair/citations/CITATION.cff \ + "${FAIR_WORKFLOW}#main" +unset INVENIO_TOKEN ``` -cwlVersion: v1.2 -$graph: -... -$namespaces: - s: https://schema.org/ -s:author: -- class: s:Person - s:affiliation: Planet Earth - s:email: john.doe@somedomain.org - s:name: Doe, John -s:contributor: -- class: s:Person - s:affiliation: Planet Earth - s:email: jane.doe@somedomain.org - s:name: Doe, Jane -s:softwareVersion: 1.1.6 -schemas: -- http://schema.org/version/9.0/schemaorg-current-http.rdf + +Without a DOI in `s:identifier`, the plugin creates a record, reserves a DOI, +uploads attachments and publishes. With an existing DOI there, it creates and +publishes a new version. Review the resulting record and its download links. +See [invenio-publish](https://github.com/transpiler-mate/invenio-publish) for the +metadata contract and versioning workflow. + +Keep the returned sandbox DOI in your practice notes. For a production deposit, +record the real identifier in the corresponding release metadata and maintain +a distinction between an individual version's DOI and a concept DOI. +`cwl2citation --doi ` can generate a fresh citation set after publication. +The deposit already uploaded above contains the original citation files; +regenerating local citations does not update that record automatically. + +Schema.org's identity-link property is case-sensitive: use `s:sameAs`, not +`s:sameas`. The citation and publication plugins use `s:identifier` for DOI +handling. Keep these fields consistent when both are present. Generating +[DataCite metadata](https://github.com/transpiler-mate/cwl2datacite) is another +export operation; it does not itself register a DOI. + +## Connect the exercise to release CI + +The current [CI workflow](../release/ci.md) already publishes packed CWL, +`cwl2oci` annotations, OCI manifests and immutable artifact references. Citation +and RO-Crate generation in this exercise remain local; CI does not yet generate +or deposit them. + +A future release job can generate citations and a Workflow RO-Crate from the +same verified packed CWL, then attach those outputs to the GitHub release. +Zenodo publication should be a separately configured publication step with +explicit credentials and a policy for identifying new versions. + +## Continue with discovery and execution provenance + +| Extension | Exercise | What it establishes | +| --- | --- | --- | +| [cwl2ogcrecords](https://github.com/transpiler-mate/cwl2ogcrecords) | Generate a record and ingest it into an OGC API – Records catalog | Searchable application metadata once cataloged | +| [cwl2ogc](https://github.com/transpiler-mate/cwl2ogc) | Generate process descriptors and JSON Schemas | A machine-readable interface for OGC API – Processes integration | +| [cwl2inputs](https://github.com/transpiler-mate/cwl2inputs) | Generate and complete an input template | A starting point for running the workflow | +| [cwl2sbom](https://github.com/transpiler-mate/cwl2sbom) | Inventory digest-pinned release images for an explicit platform | Declared container dependencies, image SBOMs and coverage information | + +SBOM generation complements CI vulnerability scanning. It does not publish or +sign its output, and it does not inventory all possible runtime downloads or +host tools. Retain the coverage report with the SBOMs. + +To describe an actual execution, capture CWLProv using your normal runner and +input settings, then convert that execution directory: + +```bash +cwltool --provenance runs/fair/execution.provenance \ + "${FAIR_WORKFLOW}#main" cwl-workflow/typed-cloud-native-inputs.yaml +transpiler-mate cwl2rocrate \ + --run runs/fair/execution.provenance \ + --output runs/fair/run-crate --zip \ + "${FAIR_WORKFLOW}#main" ``` -## Application Package identification and traceability +Run this advanced exercise only after configuring the container runtime and +input data as described in the workflow labs. Use the exact workflow that was +executed. The plugin checks the CWLProv bag and workflow match; it does not +execute the workflow or invent execution records. Recorded data may make the +result much larger than a Workflow RO-Crate. -In a Reproducible FAIR Workflow scenario, it is necessary to ensure that the Application Package is uniquely identified and traceable. +Source preservation remains a separate activity. Archive the source repository +with [Software Heritage](https://www.softwareheritage.org/) and record its SWHID +alongside the release commit, workflow digest and publication identifier. -The assignment of a DOI to the application package extends the metadata section of the Application Package CWL with: +Deactivate the tutorial environment when finished: +```bash +deactivate ``` -sameAs: URL of a reference Web page that unambiguously indicates the item’s identity. https://schema.org/sameAs. -``` - -Example - -```yaml hl_lines="16" -cwlVersion: v1.2 -$graph: -... -$namespaces: - s: https://schema.org/ -s:author: -- class: s:Person - s:affiliation: Planet Earth - s:email: john.doe@somedomain.org - s:name: Doe, John -s:contributor: -- class: s:Person - s:affiliation: Planet Earth - s:email: jane.doe@somedomain.org - s:name: Doe, Jane -s:sameas: https://doi.org/10.5072/zenodo.1107209 -s:softwareVersion: 1.1.6 -schemas: -- http://schema.org/version/9.0/schemaorg-current-http.rdf -``` \ No newline at end of file diff --git a/scripts/requirements-cwl-fair.txt b/scripts/requirements-cwl-fair.txt new file mode 100644 index 0000000..9d02990 --- /dev/null +++ b/scripts/requirements-cwl-fair.txt @@ -0,0 +1,4 @@ +-r requirements-cwl-release.txt +cwl2citation==0.1.1 +cwl2ro-crate==0.1.1 +invenio-publish==0.1.1 From c5c3f9568eb9e450da95e038d6b12dea542535db Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:45:13 +0200 Subject: [PATCH 22/23] docs(labs): record packaged and Hatch Minikube validation --- practice-labs/5-Kubernetes/README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/practice-labs/5-Kubernetes/README.md b/practice-labs/5-Kubernetes/README.md index 747a57b..189edda 100644 --- a/practice-labs/5-Kubernetes/README.md +++ b/practice-labs/5-Kubernetes/README.md @@ -32,12 +32,16 @@ Published deployments still need accessible, versioned container images and a published or otherwise distributed application package. This local test path does not publish images, a CWL release, or the working branch. -Validation on 10 October 2026 passed all 18 practice notebooks on the Docker -Minikube profile, including the direct Calrissian run, Kubernetes Job, benchmark -workflows, Gantt chart, and report tables. The tested tool snapshot was -`3292d39` plus the fixes in `361275e`, with the notebook changes documented here; -concurrent tool-packaging edits were not copied into that deployment. Remote -raster reads needed bounded retries, and an earlier Job attempt failed during -a transient external DNS outage. Final successful notebook outputs are in -`/workspace/lab-results/20261010T180151267666Z/` (Calrissian) and -`/workspace/lab-results/20261010T180519884005Z/` (benchmark). +Validation on 10 October 2026 passed all 18 practice notebooks on profile +`eoap-mastering-app-package-docker`, including the direct Calrissian run, +Kubernetes Job, both benchmark executions, Gantt chart, and report tables. +The packaged-tool snapshot was `b5275c0` with fixes `ffc625f` and `de098fb`. +The concurrent Hatch notebooks and environments were also copied into the +editor: all four chapter 1 notebooks and `task code:test:all` (29 tests) passed. +The Hatch changes remain WIP. A separate stage-out CWL roundtrip passed against +SeaweedFS, checking uploaded catalog, item, and asset content before cleanup. + +Otsu needed 1024 MiB rather than 512 MiB; normalized difference retains 2048 MiB. +Remote raster reads needed bounded retries. Successful chapter 5 notebooks are +in `/workspace/lab-results/20261010T192341432327Z/`, copied to the host at +`/tmp/mastering-metadata-packaged-kubernetes`. From 77ef29b242d2beeda0f7becd93a5188b79c90fc5 Mon Sep 17 00:00:00 2001 From: fbrito Date: Sat, 10 Oct 2026 21:48:24 +0200 Subject: [PATCH 23/23] chore(release): prepare application packages for 2.0.0 --- .github/workflows/build.yaml | 15 ++++- CHANGELOG.md | 62 ++++++++++++++++--- codemeta.json | 6 +- cwl-cli/crop.cwl | 2 +- cwl-cli/norm-diff.cwl | 2 +- cwl-cli/otsu.cwl | 2 +- cwl-cli/stac.cwl | 2 +- cwl-cli/stage-in.cwl | 2 +- cwl-cli/stage-out.cwl | 2 +- .../app-water-bodies-cloud-native.cwl | 2 +- cwl-workflow/app-water-body-cloud-native.cwl | 2 +- cwl-workflow/app-water-body.cwl | 2 +- docs/fair/best-practice.md | 4 +- .../detect_water_body.md | 4 +- .../water-bodies.md | 4 +- .../water-bodies.md | 4 +- .../app-water-body/water-bodies.md | 4 +- docs/release/ci.md | 6 ++ .../crop/src/crop/__about__.py | 2 +- .../norm_diff/src/norm_diff/__about__.py | 2 +- .../otsu/src/otsu/__about__.py | 2 +- .../stac/src/stac/__about__.py | 2 +- .../stage-in/src/stage/__about__.py | 2 +- .../stage-out/src/stage_out/__about__.py | 2 +- 24 files changed, 102 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5625cad..691758c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,6 +10,7 @@ on: - 'scripts/**' - 'tests/**' - 'codemeta.json' + - 'CHANGELOG.md' workflow_dispatch: permissions: contents: read @@ -36,6 +37,10 @@ jobs: run: | version=$(jq -er '.version' codemeta.json) [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]] + grep -Fq "## [$version]" CHANGELOG.md || { + echo "::error::CHANGELOG.md must describe release $version." + exit 1 + } registry="ghcr.io/${GITHUB_REPOSITORY,,}" echo "version=$version" >> "$GITHUB_OUTPUT" echo "registry=$registry" >> "$GITHUB_OUTPUT" @@ -198,7 +203,15 @@ jobs: path: downloads - name: Create draft, attach verified assets, and publish run: | + awk -v version="$VERSION" ' + index($0, "## [" version "]") == 1 { found=1; active=1 } + active && /^## / && index($0, "## [" version "]") != 1 { active=0 } + active { print } + END { if (!found) exit 1 } + ' CHANGELOG.md > /tmp/release-notes.md + printf '\n### OCI artifacts\n\n' >> /tmp/release-notes.md + cat downloads/oci-artifacts.txt >> /tmp/release-notes.md gh release create "$VERSION" --target "$GITHUB_SHA" --title "$VERSION" \ - --draft --notes-file downloads/oci-artifacts.txt + --draft --notes-file /tmp/release-notes.md gh release upload "$VERSION" downloads/* gh release edit "$VERSION" --draft=false diff --git a/CHANGELOG.md b/CHANGELOG.md index d8dad02..360e1cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,61 @@ ## Unreleased +## [2.0.0] - 2026-10-10 + +### Breaking changes + +- Workflow inputs now use EOAP schemas: AOIs are GeoJSON Polygon objects, + cloud-native item references are URI records, and CRS and band values use + shared named enums. Existing string and bounding-box jobs must be migrated. +- Processing tools are Python packages with console commands. Container and + workflow invocations use `crop`, `norm-diff`, `otsu`, and `stac`, replacing + direct `app.py` execution and the previous command bindings. +- Modern processing and staging packages require Python 3.12 or later. + +### Added + +- Packaged stage-in and stage-out commands, tests, wheels, and portable + multistage container images running as a non-root user. +- Taskfile commands for testing, validation, packaging previews, and docs; + Hatch environments and dedicated Bash kernels for application-step notebooks. +- Generated workflow documentation and locally rendered SVG diagrams. +- FAIR exercises for citations, Workflow RO-Crates, optional Zenodo Sandbox + publication, and execution provenance. +- Release CI gates for package tests and fixable HIGH/CRITICAL vulnerabilities; + digest-pinned packed CWL artifacts, cwl2oci annotations, and verified ORAS + publication to GHCR before GitHub release publication. + ### Changed -- Pinned pip dependencies in the crop, normalized-difference, Otsu, and STAC - application-step notebooks for ESAGEP-2379. -- Set notebook dependencies to `rasterio==1.5.2`, `click==8.5.0`, - `pystac==1.15.2`, `loguru==0.7.3`, `pyproj==3.8.0`, `shapely==2.1.2`, - `scikit-image==0.26.0`, `rio_stac==0.12.0`, `stactools[validate]==0.5.3`, - and `requests==2.34.2`, where used. +- Updated notebooks and guides for packaged commands, typed inputs, current + Calrissian jobs, benchmarking, and CLI execution from Python. +- Redesigned documentation navigation with code copying, diagram support, + image zoom, and pinned documentation dependencies. +- Aligned CodeMeta and CWL licensing with the repository's CC-BY-SA-4.0 license. +- Pinned processing dependencies, including Rasterio 1.5.2, Click 8.5.0, + PySTAC 1.15.2, Loguru 0.7.3, Shapely 2.1.2, scikit-image 0.26.0, + and rio-stac 0.12.0, as applicable to each package. + +### Fixed + +- EOAP metadata validation findings and stale documentation and notebook links. +- Crop geometry masking, bounded remote raster reads, and processing memory + requirements for normalization and Otsu. +- Stage-in failure handling and stage-out catalog copying and upload paths. +- Checked-out typed CWL packing and validation in the Kubernetes practice lab. -### Requirements +### Migration -- Notebooks using Rasterio 1.5.2 or PyProj 3.8.0 now require Python 3.12 or later. +- Start from `cwl-workflow/typed-cloud-native-inputs.yaml` for a single-item + cloud-native job. For scattering, use `stac_items` as a list of URI records + instead of the single `item` field. Supply `aoi` as a GeoJSON Polygon, + `epsg: "4326"`, and `bands: [green, nir]`. +- Install the Python packages or use the rebuilt images, and update direct + invocations to the console commands shown in the application-step labs. +- Install Hatch notebook kernels using the Taskfile instructions in + `docs/development.md`. +- Historical released CWL examples remain available with their original + interfaces and image references; they are separate from the 2.0.0 artifacts. +- Retrieve a released workflow by its OCI digest and execute its packed + `#main` entrypoint. Local source workflows select `#water-bodies`. diff --git a/codemeta.json b/codemeta.json index 2663ca6..fc47e21 100644 --- a/codemeta.json +++ b/codemeta.json @@ -5,12 +5,12 @@ "codeRepository": "https://github.com/eoap/mastering-app-package.git", "dateCreated": "2022-09-01", "datePublished": "2022-09-25", - "dateModified": "2025-09-10", + "dateModified": "2026-10-10", "name": "Water Bodies Detection", - "version": "1.1.0", + "version": "2.0.0", "description": "The Water Bodies Detection is an Application that uses the NDWI index and the Otsu threshold to detect water bodies using Sentinel-2 or Landsat-9 data", "developmentStatus": "active", - "downloadUrl": "https://github.com/eoap/mastering-app-package/releases/tag/1.1.0", + "downloadUrl": "https://github.com/eoap/mastering-app-package/releases/tag/2.0.0", "relatedLink": [ "https://eoap.github.io/mastering-app-package" ], diff --git a/cwl-cli/crop.cwl b/cwl-cli/crop.cwl index 08ce29c..614708d 100644 --- a/cwl-cli/crop.cwl +++ b/cwl-cli/crop.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool id: crop diff --git a/cwl-cli/norm-diff.cwl b/cwl-cli/norm-diff.cwl index 5642bd0..0a34db5 100644 --- a/cwl-cli/norm-diff.cwl +++ b/cwl-cli/norm-diff.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool id: norm_diff diff --git a/cwl-cli/otsu.cwl b/cwl-cli/otsu.cwl index 7ff93a4..ee42d8b 100644 --- a/cwl-cli/otsu.cwl +++ b/cwl-cli/otsu.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool id: otsu diff --git a/cwl-cli/stac.cwl b/cwl-cli/stac.cwl index 4e9b145..ba72714 100644 --- a/cwl-cli/stac.cwl +++ b/cwl-cli/stac.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool id: stac diff --git a/cwl-cli/stage-in.cwl b/cwl-cli/stage-in.cwl index abd39ae..446fe3e 100644 --- a/cwl-cli/stage-in.cwl +++ b/cwl-cli/stage-in.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool id: main diff --git a/cwl-cli/stage-out.cwl b/cwl-cli/stage-out.cwl index a0e3711..21ebd13 100644 --- a/cwl-cli/stage-out.cwl +++ b/cwl-cli/stage-out.cwl @@ -29,7 +29,7 @@ s:author: s:softwareRequirements: - container runtime - cwl runner -s:softwareVersion: 1.1.0 +s:softwareVersion: 2.0.0 class: CommandLineTool diff --git a/cwl-workflow/app-water-bodies-cloud-native.cwl b/cwl-workflow/app-water-bodies-cloud-native.cwl index b5bd774..0f7c6cb 100644 --- a/cwl-workflow/app-water-bodies-cloud-native.cwl +++ b/cwl-workflow/app-water-bodies-cloud-native.cwl @@ -1,7 +1,7 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ -s:softwareVersion: 1.4.1 +s:softwareVersion: 2.0.0 # Training-example authors and creation date mirror codemeta.json. s:name: Water bodies detection based on NDWI and otsu threshold s:description: Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items diff --git a/cwl-workflow/app-water-body-cloud-native.cwl b/cwl-workflow/app-water-body-cloud-native.cwl index 11afeb5..314c8d0 100644 --- a/cwl-workflow/app-water-body-cloud-native.cwl +++ b/cwl-workflow/app-water-body-cloud-native.cwl @@ -1,7 +1,7 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ -s:softwareVersion: 1.0.0 +s:softwareVersion: 2.0.0 # Training-example authors and creation date mirror codemeta.json. s:name: Water bodies detection based on NDWI and the otsu threshold s:description: Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item diff --git a/cwl-workflow/app-water-body.cwl b/cwl-workflow/app-water-body.cwl index 1e669cb..b09f5d6 100644 --- a/cwl-workflow/app-water-body.cwl +++ b/cwl-workflow/app-water-body.cwl @@ -1,7 +1,7 @@ cwlVersion: v1.2 $namespaces: s: https://schema.org/ -s:softwareVersion: 1.0.0 +s:softwareVersion: 2.0.0 # Training-example authors and creation date mirror codemeta.json. s:name: Water body detection based on NDWI and the otsu threshold s:description: Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or Landsat-9 staged acquisitions diff --git a/docs/fair/best-practice.md b/docs/fair/best-practice.md index 42dbfc9..6624aa7 100644 --- a/docs/fair/best-practice.md +++ b/docs/fair/best-practice.md @@ -61,8 +61,8 @@ Check the following before a real publication: can retain their individual development versions; the release preparation job sets the version and repository on copies before packing. Generate release citations and crates from those final copies, so all published outputs agree. -The current `1.1.0` CodeMeta version is already released and must be bumped for -a new publication through CI. +The prepared release version is `2.0.0`. Subsequent releases must use a new +version; CI rejects existing tags and published versions. The [release tutorial](../release/ci.md) explains that process. Keep generated CodeMeta separate from the CI input: the diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md index fce9ef5..1c52d45 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/detect_water_body.md @@ -1,10 +1,10 @@ -# Water bodies detection based on NDWI and otsu threshold v1.4.1 +# Water bodies detection based on NDWI and otsu threshold v2.0.0 Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T20:29:45.005 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T21:46:39.161 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team diff --git a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md index 4f84b64..39d874d 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-bodies-cloud-native/water-bodies.md @@ -1,10 +1,10 @@ -# Water bodies detection based on NDWI and otsu threshold v1.4.1 +# Water bodies detection based on NDWI and otsu threshold v2.0.0 Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 COG STAC items > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T20:29:45.026 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T21:46:39.133 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team diff --git a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md index 2cd8239..2fa20d4 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body-cloud-native/water-bodies.md @@ -1,10 +1,10 @@ -# Water bodies detection based on NDWI and the otsu threshold v1.0.0 +# Water bodies detection based on NDWI and the otsu threshold v2.0.0 Water bodies detection based on NDWI and otsu threshold applied to a single Sentinel-2 COG STAC item > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T20:29:47.518 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T21:46:41.916 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team diff --git a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md index 947cee8..8171cc2 100644 --- a/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md +++ b/docs/reference/cwl/cwl-workflow/app-water-body/water-bodies.md @@ -1,10 +1,10 @@ -# Water body detection based on NDWI and the otsu threshold v1.0.0 +# Water body detection based on NDWI and the otsu threshold v2.0.0 Water bodies detection based on NDWI and otsu threshold applied to Sentinel-2 or Landsat-9 staged acquisitions > This software is licensed under the terms of the [Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) license - SPDX short identifier: [CC-BY-SA-4.0](https://spdx.org/licenses/CC-BY-SA-4.0) > -> 2022-09-01 - 2026-10-10T20:29:49.248 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) +> 2022-09-01 - 2026-10-10T21:46:43.796 Copyright [EO Application Packaging](mailto:None) - > [https://github.com/eoap](https://github.com/eoap) ## Project Team diff --git a/docs/release/ci.md b/docs/release/ci.md index fd09460..621fcfb 100644 --- a/docs/release/ci.md +++ b/docs/release/ci.md @@ -91,3 +91,9 @@ and `contents: write` only for release creation. GHCR must allow the repository' token to publish these packages. A failure after image or OCI publication can leave registry artifacts without a GitHub release; inspect the failed run and use a new version for the next release. Registry publication is not transactional. + +For the prepared `2.0.0` release, the changelog documents the typed-input and +console-command migration. The release job requires a matching version section +in `CHANGELOG.md` and uses it as the GitHub release notes, followed by the +verified OCI artifact references. Package versions and current CWL metadata are +aligned to `2.0.0`; historical released CWL examples retain their old metadata. diff --git a/water-bodies/command-line-tools/crop/src/crop/__about__.py b/water-bodies/command-line-tools/crop/src/crop/__about__.py index e5abda9..46cc37d 100644 --- a/water-bodies/command-line-tools/crop/src/crop/__about__.py +++ b/water-bodies/command-line-tools/crop/src/crop/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2026-present U.N. Owen # # SPDX-License-Identifier: MIT -__version__ = "0.0.1" +__version__ = "2.0.0" diff --git a/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py index f102a9c..8c0d5d5 100644 --- a/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py +++ b/water-bodies/command-line-tools/norm_diff/src/norm_diff/__about__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "2.0.0" diff --git a/water-bodies/command-line-tools/otsu/src/otsu/__about__.py b/water-bodies/command-line-tools/otsu/src/otsu/__about__.py index f102a9c..8c0d5d5 100644 --- a/water-bodies/command-line-tools/otsu/src/otsu/__about__.py +++ b/water-bodies/command-line-tools/otsu/src/otsu/__about__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "2.0.0" diff --git a/water-bodies/command-line-tools/stac/src/stac/__about__.py b/water-bodies/command-line-tools/stac/src/stac/__about__.py index f102a9c..8c0d5d5 100644 --- a/water-bodies/command-line-tools/stac/src/stac/__about__.py +++ b/water-bodies/command-line-tools/stac/src/stac/__about__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "2.0.0" diff --git a/water-bodies/command-line-tools/stage-in/src/stage/__about__.py b/water-bodies/command-line-tools/stage-in/src/stage/__about__.py index f102a9c..8c0d5d5 100644 --- a/water-bodies/command-line-tools/stage-in/src/stage/__about__.py +++ b/water-bodies/command-line-tools/stage-in/src/stage/__about__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "2.0.0" diff --git a/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py b/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py index f102a9c..8c0d5d5 100644 --- a/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py +++ b/water-bodies/command-line-tools/stage-out/src/stage_out/__about__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "2.0.0"