Merge pull request #162 from TIDES-transit/sample-data-doc

Sample Data: document, update scripts to validate, update tests

Author: SorenSpicknall

.github/workflows/validate_package_schema.yml

@@ -17,7 +17,7 @@ jobs:
         uses: TIDES-transit/json-schema-validator@master
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          json_schema: spec/tides-data-package.json
+          json_schema: spec/tides-datapackage-profile.json
           json_path_pattern: ^spec/tides\.spec\.json$
           send_comment: false
           clear_comments: false

.github/workflows/validate_samples.yml

@@ -29,31 +29,47 @@ jobs:
         run: |
           pwd
           samples=$(find ./samples -type d -name 'TIDES')
-          echo "{name}={value}" >> $GITHUB_OUTPUT
+          echo $samples
+          echo "samples=${samples}" >> $GITHUB_OUTPUT
       - name: Validate Sample Data
         id: validation
         run: |
           samples=(${{ steps.sample_folders.outputs.samples }})
-          results=""
+          results="{\"values\": ["
+          result=""
           for sample in "${samples[@]}"; do
-            result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json)
-            results+="$result\n"
+            results+="$result"
+            result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true
+            result="${result//\'s/}"
+            result="${result//\\\"/\\\\\"}"
+            result="$result,"
           done
-          echo "{name}=${results}" >> $GITHUB_ENV
+
+          result="${result%?}"
+          results+="$result"
+          results+="]}"
+
+          echo "RESULTS<<EOF" >> $GITHUB_OUTPUT
+          echo ${results} >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
       - name: Comment on PR
         if: github.event_name == 'pull_request'
         uses: actions/github-script@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const samples = '${{ env.samples }}'.trim().split('\n');
-            const results = '${{ env.results }}'.trim().split('\n');
+            console.log("Begin sample construction")
+            const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n');
+            const results = JSON.parse('${{ steps.validation.outputs.results }}');
             let comment = `**Data Validation Report**\n\n`;
             comment += '| Sample | Status |\n';
             comment += '| ------ | ------ |\n';
+            console.log("Constructed header")
             for (let i = 0; i < samples.length; i++) {
+              console.log(samples[i])
+              console.log(results[i])
               const sample = samples[i];
-              const result = JSON.parse(results[i]);
+              const result = results.values[i];
               let status = '';
               if (result.valid) {
                 status = ':heavy_check_mark:';

.github/workflows/validate_table_schemas.yml

@@ -21,4 +21,4 @@ jobs:
           json_schema: spec/table-schema.json
           json_path_pattern: ^spec/.*\.schema\.json$
           send_comment: false
-          clear_comments: false
+          clear_comments: false

CONTRIBUTING.md

@@ -34,9 +34,10 @@ By making any contribution to the projects, contributors self-certify to the [Co
 
 1. [Create a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) to work on a new issue (or checkout an existing one where the issue is being worked on).  
 2. Make your changes.
-3. Run `/tests/test_all` script to check and fix formatting, validate schemas, and build documentation locally to preview
-4. [Commit](#commits) your work in `git`
-5. `push` your changes to Github and submit a [`pull request`](#pull-requests)
+3. Run `tests/test_local_spec` script to check and fix formatting, validate profile and schemas with frictionless and with each other, and confirm that documentation can be built locally.
+4. Run `tests/test_samples_to_local` script to check if samples conform to any changes to the spec.
+5. [Commit](#commits) your work in `git`
+6. `push` your changes to Github and submit a [`pull request`](#pull-requests)
 
 ### Issues
 
@@ -106,7 +107,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de
     | **Name** |  **What it does** |
     | -------- | ----------------- |
     | GitHub Actions | Runs following workflow on each push to the TIDES github repository:  /.github/workflows/docs.yml |
-    | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc) <br> For new branches with documentation, adds an entry in `versions.json` | 
+    | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc) <br> For new branches with documentation, adds an entry in `versions.json` |
     | `mkdocs` | Package which generates documentation from markdown and code |
 
 ??? info "Overview of Documentation Building Process"
@@ -117,7 +118,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de
     subgraph mkdocs["<b>mkdocs:</b> run on execution of mike"]
     md_mike["mike"] -->|runs for current branch| md_mkdocs["mkdocs"]
     md_mkdocs.yml["mkdocs.yml"] -->|specifieds parameters| md_mkdocs["mkdocs"]
-    md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"] 
+    md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"]
     main.py[/"main.py"/] -->|defines macros in code available for| md_mkdocs_macros["mkdocs-macros"]
     end
 

README.md

@@ -12,6 +12,35 @@ Human-friendlier documentation is auto-generated and available at:
 - [Architecture](https://tides-transit.github.io/TIDES/main/architecture)
 - [Table Schemas](https://tides-transit.github.io/TIDES/main/tables)
 
+### Data Package
+
+Directories with TIDES data must contain metadata in a [`datapackage.json`](https://tides-transit.github.io/TIDES/main/datapackage) file in a format compliant with the [`tides-datapackage-profile`](https://tides-transit.github.io/TIDES/main/datapackage) of a [`frictionless data package`](https://specs.frictionlessdata.io/data-package/).  
+
+[`/samples/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/datapackage.json) has a template datapackage which can be used.
+
+## Sample Data
+
+[Sample data](https://tides-transit.github.io/TIDES/main/samples) can be found in the `/samples` directory, with one directory for each sample.
+
+### Template
+
+Templates of `datapackage.json` and each TIDES file type are located in the `/samples/template` directory.
+
+## Validating TIDES data
+
+TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows:
+
+```bash
+pip install frictionless
+frictionless validate --schema-sync path/to/your/datapackage.json
+```
+
+Several other validation scripts and tools with more flexibility such as validating to the canonical, named version or a local spec can be found in the `/bin` directory, with usage available with the `--help` flag.
+
+```bash
+bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path]
+```
+
 ## Contributing to TIDES
 
 Those who want to help with the development of the TIDES specification should review the guidance in [CONTRIBUTING.md](CONTRIBUTING.md).

bin/replace-spec-in-datapackage

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+description="Create a temporary data package if a spec_path_prefix is provided."
+
+usage="
+Usage: replace-spec-in-datapackage <dataset_path> [spec_path_prefix] [output_file]
+
+Arguments:
+  <dataset_path>   The path to the dataset directory containing the 'datapackage.json' file.
+  <spec_path_prefix>   The path or URL to the spec to be referenced in the updated data package.
+  [output_file]   (Optional) The path to save the temporary data package. If not provided, the temporary data package will be saved as 'datapackage.tmp.json' in the dataset directory.
+
+"
+
+example_usage="
+Example Usage: 
+bin/replace-spec-in-datapackage samples/template/TIDES spec samples/template/TIDES/datapackage.tmp.json
+"
+
+################################################################################
+# Help                                                                         #
+################################################################################
+
+# Display help message
+function display_help() {
+  echo "$description"
+  echo "$usage"
+  echo "$example_usage"
+}
+
+# Check for help flag
+if [ "$1" == "--help" ]; then
+  display_help
+  exit 0
+fi
+
+################################################################################
+# SET DEFAULTS                                                                 #
+################################################################################
+DEFAULT_TMP_DATAPACKAGE="datapackage.tmp.json"
+PROFILE_FILE="tides-datapackage-profile.json"
+
+################################################################################
+# MAIN                                                                         #
+################################################################################
+
+echo "$description"
+
+################################################################################
+# Check Requirements                                                           #
+################################################################################
+source "$(dirname "${BASH_SOURCE[0]}")/utils"
+check_jq
+
+################################################################################
+# Process the input options.                                                   #
+################################################################################
+dataset_path=$1
+spec_path_prefix=$2
+output_file=${3:-"$dataset_path/$DEFAULT_TMP_DATAPACKAGE"}
+
+echo "Parameters:
+    dataset_path: $dataset_path
+    spec_path_prefix: $spec_path_prefix
+    output_file: $output_file
+"
+
+# Check if required arguments are missing
+if [ -z "$dataset_path" ]; then
+  echo "Error: Missing dataset_path argument." >&2
+  display_help
+  exit 1
+fi
+
+# Check if required arguments are missing
+if [ -z "spec_path_prefix" ]; then
+  echo "Error: Missing spec_path_prefix argument." >&2
+  display_help
+  exit 1
+fi
+
+datapackage_file="$dataset_path/datapackage.json"
+
+check_valid_path "$datapackage_file"
+
+profile_file="$spec_path_prefix/$PROFILE_FILE"
+check_valid_path "$profile_file"
+
+################################################################################
+# Create updated datapackage                                                   #
+################################################################################
+cp "$datapackage_file" "$output_file"
+jq --arg spec_path_prefix "$spec_path_prefix" --arg profile_file "$profile_file" '
+  .resources |= map(.schema |= ($spec_path_prefix + "/\(. | split("/") | last)"))
+  | .profile = ($profile_file)
+' "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file"
+
+echo "$output_file"

bin/utils

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# Check if jsonschema-cli is installed
+check_jsonschema-cli() {
+  if ! command -v jsonschema-cli >/dev/null 2>&1; then 
+    echo >&2 "\033[31m!!! jsonschema-cli is required but not found.\033[0m 
+    
+    You can install it using 'pip install jsonschema-cli'. Aborting."
+    exit 1
+  fi
+}
+
+# Check if frictionless is installed
+check_frictionless() {
+  if ! command -v frictionless >/dev/null 2>&1; then 
+    echo >&2 "\033[31m!!! frictionless is required but not found.\033[0m 
+    
+    You can install it using 'pip install frictionless'. Aborting."
+    exit 1
+  fi
+}
+
+# Function to check jq and provide installation instructions if not found
+check_jq() {
+  if ! command -v jq >/dev/null 2>&1; then
+    echo >&2 "\033[31m!!! jq is required but not found.\033[0m"
+
+    # Determine the operating system
+    os_type=$(uname -s)
+
+    case "$os_type" in
+      Linux*)
+        echo >&2 "Please install jq using your package manager."
+        echo >&2 "For example, on Debian/Ubuntu-based systems, you can use:"
+        echo >&2 "sudo apt-get install jq"
+        ;;
+      Darwin*)
+        echo >&2 "Please install jq using Homebrew."
+        echo >&2 "If you don't have Homebrew installed, you can install it from https://brew.sh/."
+        echo >&2 "Once Homebrew is installed, run the following command:"
+        echo >&2 "brew install jq"
+        ;;
+      CYGWIN*|MINGW32*|MSYS*|MINGW*)
+        echo >&2 "Please download the jq binary for Windows from https://stedolan.github.io/jq/download/."
+        echo >&2 "Extract the downloaded ZIP file and add the 'jq.exe' binary to your PATH..."
+        echo >&2 "Or run the following command: "
+        echo >&2 "curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe"
+        ;;
+      *)
+        echo >&2 "Unknown operating system. Please install jq from https://stedolan.github.io/jq/download/."
+        ;;
+    esac
+
+    exit 1
+  fi
+}
+
+# Function to check if this is a valid spec path
+check_valid_path() {
+  local path=$1
+
+  if [ -n "$path" ]; then
+    if [ ! -e "$path" ] && [[ ! "$path" =~ ^http ]]; then
+      echo "\033[31mError: $path is an invalid path. It must be a valid file path, directory path, or a URL starting with 'http'.\033[0m" >&2
+      display_help
+      exit 1
+    elif [[ "$path" =~ ^http ]]; then
+      # Check if the URL is valid
+      if ! curl --output /dev/null --silent --head --fail "$path"; then
+        echo "\033[31mError: Invalid URL. The $path URL is not reachable or does not exist.\033[0m" >&2
+        exit 1
+      fi
+    fi
+  fi
+}