Refactoring of training and prediction code. Added more tests.

Author: mneagul

.travis.yml

@@ -1,9 +1,9 @@
 language: python
 matrix:
   include:
-    - name: "Python 3.7 on Xenial Linux"
+    - name: "Python 3.7 on Bionic Linux"
       python: 3.7
-      dist: xenial
+      dist: bionic
       env:
         - PIP_CMD=pip3
         - PYTHON=python3
@@ -17,20 +17,14 @@ matrix:
       env:
         - PIP_CMD=pip3
         - PYTHON=python3
-    - name: "Python 3.6 on Xenial Linux"
+    - name: "Python 3.6 on Bionic Linux"
       python: 3.6
-      dist: xenial
-      env:
-        - PIP_CMD=pip3
-        - PYTHON=python3
-    - name: "Python 3.5 on Xenial Linux"
-      python: 3.5
-      dist: xenial
+      dist: bionic
       env:
         - PIP_CMD=pip3
         - PYTHON=python3
 install:
   - $PIP_CMD install --upgrade pip
   - $PIP_CMD install -r requirements.txt
   - $PIP_CMD install .
-script: $PYTHON -m pytest
+script: $PYTHON -m pytest -v -s

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/examples/predictv3.yaml

@@ -0,0 +1,50 @@
+data_source: !!python/object/apply:hugin.io.FileSystemLoader
+  kwds:
+    data_pattern: '(?P<name>[0-9A-Za-z_]+)__(?P<idx>[A-Za-z0-9_\.]+)_(?P<type>B.*)\.tif$'
+    id_format: '{name}-{idx}'
+    type_format: '{type}'
+    input_source: /Users/mneagul/Downloads/S2A_OPER_MSI_L1C_TL_SGS__20170519T130610_A009957_T34TFQ_N02.05/data/rezolutie_nativa # Change to the path to the prediction data (can be changed at runtime)
+
+
+predictor: !!python/object/apply:hugin.infer.scene.AvgEnsembleScenePredictor
+  kwds:
+    name: interesting-ensemble
+    predictors:
+      - predictor: !!python/object/apply:hugin.infer.scene.RasterScenePredictor
+          kwds:
+            name: raster_predictor1
+            model: !!python/object/apply:hugin.infer.core.KerasPredictor
+              kwds:
+                name: keras_predictor1
+                model_path: /Users/mneagul/Work/Temp/mdl/forestry_training-09.hdf5
+                model_builder: hugin.models.unet.unetv14:unet_v14
+                input_shape: [256, 256]
+            mapping:
+              inputs:
+                - [ "B02_10m", 1 ]
+                - [ "B03_10m", 1 ]
+                - [ "B04_10m", 1 ]
+                - [ "B08_10m", 1 ]
+              target:
+                - [ "B01_10m_GTI", 1]
+            prediction_merger: !!python/name:hugin.infer.core.AverageMerger
+        weight: 1
+    post_processors:
+      #- !!python/name:hugin.engine.core.identity_processor
+      - !!python/object/apply:hugin.infer.core.CategoricalConverter
+          kwds:
+            num_classes: 6
+    metrics:
+      metric_1: !!python/name:hugin.infer.core.identity_metric
+      metric_2: !!python/name:hugin.infer.core.identity_metric
+    gti_component: 'B01_10m_GTI'
+
+output: !!python/object/apply:hugin.infer.scene.RasterIOSceneExporter
+  kwds:
+    destination: "/Users/mneagul/Work/Temp/export/"
+    metric_destination: "/Users/mneagul/Work/Temp/export/metrics.txt"
+    srs_source_component: "B02_10m"
+    rasterio_creation_options:
+      blockxsize: 256
+      blockysize: 256
+

docs/examples/trainv2.yaml

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/make.bat

@@ -0,0 +1,59 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'Hugin EO'
+copyright = '2019, HuginEO Contributors'
+author = 'Marian Neagul, Teodora Selea, Gabriel Iuhasz, Alexandru Munteanu'
+
+# The full version, including alpha/beta/rc tags
+release = '0.2.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+	'recommonmark',
+	'sphinx_rtd_theme'
+]
+
+master_doc = 'index'
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

docs/source/conf.py

@@ -0,0 +1,29 @@
+==========================================================
+Hugin EO: Machine Learning for Earth Observation made easy
+==========================================================
+
+Hugin helps scientists run Machine Learning experiments on geospatial raster data.
+
+Hugin was developed as part of the ESA funded ML4EO_
+
+Overall Hugin aims to facilitate experimentation with multiple machine learning problems, like:
+ * Classification
+ * Segmentation
+ * Super-Resolution
+
+Currently Hugin builds on top of the Keras machine learning library but it also aims to support, in the future, additional backends like scikit-learn.
+
+.. toctree::
+   :maxdepth: 2
+
+   intro
+   installation
+   quickstart
+   topics/index
+   api/index
+   contributing
+   faq
+
+
+
+.. _ML4EO: http://sage.ieat.ro/projects/ML4EO/

docs/source/index.rst

@@ -0,0 +1,2 @@
+Hugin EO Installation
+=====================

docs/source/installation.rst

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+fswatch -o .. | xargs -n1 make html

docs/watch.sh

@@ -0,0 +1,3 @@
+python:
+  version: 3
+  pip_install: true

readthedocs.yml

@@ -15,9 +15,8 @@
     'Fiona',
     'h5py',
     'backoff',
-    'matplotlib<3.1',
+    'matplotlib',
     'scikit-learn',
-    'imgaug'
 ]
 
 def extra_files(directory):
@@ -37,5 +36,8 @@ def extra_files(directory):
     entry_points = {
         'console_scripts': ['hugin=hugin.tools.cli:main'],
     },
-    description='Hugin ML4EO experimentation tool'
+    description='Hugin ML4EO experimentation tool',
+    extras_require = {
+        'hugin_data_augmentation': ["imgaug"]
+    }
 )

setup.py

@@ -2,16 +2,17 @@ version: 1
 formatters:
   brief:
     format: '%(message)s'
-  default:
+  hugindefault:
     format: "%(asctime)s %(filename)12s:%(lineno)d\t%(levelname)-8s %(name)-18s %(message)s"
     datefmt: '%Y-%m-%d %H:%M:%S'
 handlers:
   console:
     class : logging.StreamHandler
-    formatter: default
+    formatter: hugindefault
     level   : DEBUG
     stream  : ext://sys.stderr
 loggers:
   hugin:
     handlers: [console, ]
     level: DEBUG
+