Devel

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.github/workflows/format-check.yml

@@ -30,8 +30,8 @@ jobs:
           sudo apt-get install -y python3-sphinx sphinx-rtd-theme-common
           pip install sphinx_rtd_theme recommonmark flake8 black pytest pytest-cov black[jupyter]
       - name: Black formatting checks
-        run: black  --exclude '/jupyter_notebooks/' --check .
+        run: black --line-length 88 --exclude '/jupyter_notebooks/' --check .
       - name: Lint with flake8
         run: |
-          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length 100 .
-          flake8 --count --statistics --max-line-length 100 .
+          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics .
+          flake8 --count --statistics .

.github/workflows/jupyternotebook-lint.yml

@@ -0,0 +1,40 @@
+name: Jupyter Notebook Linter
+on:
+  push:
+    paths:
+      - '**/*.ipynb'
+      - '.github/workflows/jupyternotebook-lint.yml'
+
+jobs:
+  lint-jupyter:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'  # Specify the Python version
+
+      # Step 3: Install dependencies
+      - name: Install nbQA, flake8, and black
+        run: |
+          pip install nbqa flake8 black
+
+      # Step 4: Run flake8 and black on notebooks in ./jupyter_notebooks folder
+      - name: Lint Jupyter Notebooks with flake8
+        run: nbqa flake8 .
+        working-directory: jupyter_notebooks
+
+      - name: Format Jupyter Notebooks with black
+        run: nbqa black . --check
+        working-directory: jupyter_notebooks
+
+      # Step 5: Report status
+      - name: Complete
+        run: echo "Jupyter Notebook linting completed!"
+

.gitlab/build/build_gcs_image.yml

@@ -26,6 +26,8 @@ build-gcs:
         - cmake/**/*
         - repository/docker/entrypoint_authz.sh
         - repository/docker/Dockerfile.gcs
+        - repository/CMakeLists.txt
+        - repository/gridftp/**/*
       when: on_success
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
@@ -70,5 +72,7 @@ retag-image:
         - cmake/**/*
         - repository/docker/entrypoint_authz.sh
         - repository/docker/Dockerfile.gcs
+        - repository/CMakeLists.txt
+        - repository/gridftp/**/*
       when: never
     - when: on_success

.gitlab/build/build_repo_image.yml

@@ -22,9 +22,12 @@ build-repo:
     - changes:
         - docker/**/*
         - scripts/**/*
-        - web/**/*
         - common/proto/**/*
         - .gitlab-ci.yml
+        - CMakeLists.txt
+        - cmake/**/*
+        - repository/CMakeLists.txt
+        - repository/server/**/*
       when: on_success
 
 retag-image:
@@ -42,9 +45,12 @@ retag-image:
     - changes:
         - docker/**/*
         - scripts/**/*
-        - web/**/*
         - common/proto/**/*
         - .gitlab-ci.yml
+        - CMakeLists.txt
+        - cmake/**/*
+        - repository/CMakeLists.txt
+        - repository/server/**/*
       when: never
     - when: on_success
 

CHANGELOG.md

@@ -29,10 +29,12 @@
 17. [1168] - Add authz unit testing to the CI
 18. [1200] - Add JavaScript linter (eslint) and (prettier) formatter for JavaScript
 19. [1180] - Refactor of authz foxx module, split into objects and added unit tests
-20. [1257] - Bug in authz library parsing of authz.conf file, globus paths incorrectly sanitized when using just '/'
-21. [1255] - Fixes bug, in libauthz with log_path var not being read in from config file.
-22. [1268] - Bug fixes unbound variable in foxx entrypoint file.
-23. [1269] - Update Arangodb to 3.12.4
+20. [1223] - Fix deprecated method usage for Protobuf in Python client library
+21. [1257] - Bug in authz library parsing of authz.conf file, globus paths incorrectly sanitized when using just '/'
+22. [1255] - Fixes bug, in libauthz with log_path var not being read in from config file.
+23. [1268] - Bug fixes unbound variable in foxx entrypoint file.
+24. [1269] - Update Arangodb to 3.12.4
+25. [1288] - Bug Jupyternote books in documentation were fixed.
 
 # v2024.6.17.10.40
 

doc_source/source/user/web/portal.rst

@@ -97,6 +97,175 @@ Search
 
 The ``Search`` tab below the action buttons allows users to query DataFed for data matching a specified metadata expression within a specified scope.
 
+Schemas
+-------
+
+DataFed currently supports schema creation for records through the web portal.
+
+Schema Creation
+^^^^^^^^^^^^^^^
+
+To create a new schema, navigate to the main dashboard and click the ``Schemas`` button. This will open the ``Create New Schema`` dialog, which consists of three separate tabs: ``General``, ``Definition``, and ``References``.
+
+- **General Tab**:  
+  In this tab, you must provide a valid string identifier (`ID`) and a brief `Description` of the schema.
+
+- **Definition Tab**:  
+  This tab requires a **valid JSON Schema definition**. DataFed conforms to the official [JSON Schema standard](https://json-schema.org/).
+
+- **References Tab**:  
+  This section lists locations where the schema is referenced within DataFed.
+
+### Example Schema Definition
+
+The following is an example of a JSON schema definition used in DataFed:
+
+.. code-block:: json
+   {
+       "properties": {
+           "Cat": {
+               "properties": {
+                   "Height": {
+                       "type": "string"
+                   },
+                   "LastSeen_DateTime": {
+                       "type": "string"
+                   },
+                   "Name": {
+                       "type": "string"
+                   },
+                   "Photo_Date": {
+                       "type": "string"
+                   },
+                   "Weight": {
+                       "type": "string"
+                   }
+               },
+               "required": [
+                   "Name",
+                   "Photo_Date",
+                   "Height",
+                   "Weight",
+                   "LastSeen_DateTime"
+               ],
+               "type": "object"
+           },
+           "Owner": {
+               "properties": {
+                   "Address": {
+                       "type": "string"
+                   },
+                   "Name": {
+                       "type": "string"
+                   }
+               },
+               "required": [
+                   "Name",
+                   "Address"
+               ],
+               "type": "object"
+           }
+       },
+       "required": [
+           "Cat",
+           "Owner"
+       ],
+       "type": "object"
+   }
+
+### Example Schema Using Enums
+
+The following example demonstrates the use of **enums** to enforce predefined choices for various fields:
+
+.. code-block:: json
+   {
+       "properties": {
+           "data_format": {
+               "description": "The format in which experimental data is stored.",
+               "enum": [
+                   "CSV",
+                   "JSON",
+                   "HDF5",
+                   "NetCDF"
+               ],
+               "type": "string"
+           },
+           "experiment_type": {
+               "description": "The type of scientific experiment being conducted.",
+               "enum": [
+                   "Physics",
+                   "Chemistry",
+                   "Biology",
+                   "Astronomy",
+                   "Materials Science"
+               ],
+               "type": "string"
+           },
+           "instrument": {
+               "description": "The scientific instrument used for measurements.",
+               "enum": [
+                   "Spectrometer",
+                   "Microscope",
+                   "X-ray Diffractometer",
+                   "Mass Spectrometer",
+                   "Electron Microscope"
+               ],
+               "type": "string"
+           },
+           "measurement_unit": {
+               "description": "The SI unit of measurement used in the experiment.",
+               "enum": [
+                   "meters",
+                   "kilograms",
+                   "seconds",
+                   "kelvin",
+                   "moles",
+                   "amperes",
+                   "candela"
+               ],
+               "type": "string"
+           },
+           "status": {
+               "description": "The current status of the experiment.",
+               "enum": [
+                   "Pending",
+                   "In Progress",
+                   "Completed",
+                   "Failed"
+               ],
+               "type": "string"
+           }
+       },
+       "required": [
+           "experiment_type",
+           "measurement_unit",
+           "instrument",
+           "status",
+           "data_format"
+       ],
+       "type": "object"
+   }
+
+### Example Schema Using References
+
+The following example demonstrates the use of **references** to existing schemas, assuming the previous schema was saved as **equipment** with verison number 0 **equipment:0**:
+
+.. code-block:: json
+   {
+       "properties": {
+           "equipment": {
+               "$ref": "equipment:0"
+           },
+           "voltage": {
+               "type": "number"
+           }
+       },
+       "required": [
+           "voltage",
+           "equipment"
+       ],
+       "type": "object"
+   }
 
 Transfers
 ---------

jupyter_notebooks/.flake8

@@ -0,0 +1,12 @@
+[flake8]
+max-line-length = 100
+
+# Ignoring the following is specific to jupyternotebooks
+#
+# F821 - Ignore undefined variable errors, these are false positives in jupyter
+#        notebooks if they are defined in a previous cell.
+# F841 - assigned to but unused, appears because the examples have code
+#        that the user will need to fill in.
+# F401 - imported but unused, occurs because the user will need to fill in
+#        parts that require the import.
+extend-ignore = F821, F401, F841