various updates

Author: mneagul

etc/forestry/forestry-s3-v2.yaml

@@ -1,5 +1,5 @@
 configuration:
-  model_path: "s3://sagegroup-data/sage/models/{name}"
+#  model_path: "s3://sagegroup-data/sage/models/{name}"
   name: "unet-forestry-dev"
   workspace: "/home/marian/models/{name}"
 
@@ -15,6 +15,7 @@ data_source: !!python/object/apply:hugin.io.FSSpecFilesystemLoader
     type_format: '{type}'
     validation_percent: 0.25
     randomise: True
+    persist_file: ""
 
 trainer: !!python/object/apply:hugin.engine.scene.RasterSceneTrainer
          kwds:
@@ -54,18 +55,11 @@ trainer: !!python/object/apply:hugin.engine.scene.RasterSceneTrainer
              model_builder_options:
                output_channels: 2
              model_path: "/data/bid/storage0/sage-storage/homes/eosmith/marian/models/{name}"
-             batch_size: 16
+             batch_size: 14
              epochs: 100
              metrics:
                - categorical_accuracy
                - !!python/name:hugin.tools.utils.dice_coef
-               - !!python/object/apply:tensorflow.keras.metrics.MeanIoU
-                 kwds:
-                   num_classes: 2
-               - !!python/object/apply:tensorflow.keras.metrics.IoU
-                 kwds:
-                   target_class_ids: [0, 1]
-                   num_classes: 2
                - !!python/object/apply:tensorflow.keras.metrics.BinaryIoU
                  kwds:
                    target_class_ids: [1]

requirements.txt

@@ -1,21 +1,18 @@
-numpy>=1.16.0,<1.19.0
-geojson==2.4.1
+numpy
+geojson
 rasterio
 scikit-image
 scipy
 geopandas
 Shapely
 Fiona
-s3fs
 dask[array]
-gcsfs
-zarr>=2.6.0 # Needed due to bug in S3FS interaction, see: https://github.com/zarr-developers/zarr-python/issues/649
-h5py==2.10.0
-backoff==1.8.0
+zarr
+h5py
+backoff
 matplotlib
 scikit-learn
-imgaug==0.2.8
-tensorflow>=1.15,<=2.3.1
-backports.tempfile==1.0
+imgaug
+tensorflow
 tqdm
 traitlets

src/hugin/io/dataset_loaders.py

@@ -161,6 +161,15 @@ def __init__(self,
         else:
             self.scan_datasets()
 
+    @property
+    def persist_file(self):
+        if not hasattr(self, "_persist_file"):
+            return None
+        else:
+            return self._persist_file
+    @persist_file.setter
+    def persist_file(self, value):
+        self._persist_file = value
 
     def scan_datasets(self):
         self._datasets.clear()

src/hugin/tools/train.py

@@ -1,3 +1,22 @@
+# -*- coding: utf-8 -*-
+
+
+__license__ = \
+    """Copyright 2023 West University of Timisoara
+
+       Licensed under the Apache License, Version 2.0 (the "License");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+
+           http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an "AS IS" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+    """
+
 import logging
 from urllib.parse import urlparse, parse_qs
 
@@ -45,11 +64,12 @@ def train_handler(args):
 
     if workspace_directory is not None:
         workspace_directory = workspace_directory.format(**experiment_configuration)
-
+    else:
+        raise ValueError("No workspace directory specified")
     if not os.path.exists(workspace_directory):
         os.makedirs(workspace_directory)
 
-    title = f"hugin train (name: {experiment_name})"
+    title = f"hugin train ({experiment_name})"
     setproctitle(title)
     log.info(f"workspace directory: {workspace_directory}")
     if workspace_directory and trainer.base_directory is None: