Merge remote-tracking branch 'origin/master' into cuda

Author: BerndDoser

.devcontainer/Dockerfile

@@ -19,10 +19,13 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
         libsm6 \
         libxext6 \
         moreutils \
+        ninja-build \
+        patchelf \
         python3 \
         python3-dev \
         python3-pip \
         python3-tk \
+        python3-venv \
         software-properties-common \
         sudo \
         vim \
@@ -33,10 +36,18 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 RUN pip install --upgrade pip \
  && hash -r pip3 \
  && pip install \
+        auditwheel \
+        build \
         cmake \
         conan==1.* \
+        ipympl \
+        ipykernel \
+        ipywidgets \
+        h5py \
         matplotlib \
-        numpy
+        numpy \
+        twine \
+        wheel
 
 ARG USERNAME=vscode
 ARG USER_UID=15005

.devcontainer/devcontainer.json

@@ -15,8 +15,14 @@
 	"customizations": {
 		"vscode": {
 			"extensions": [
-				"ms-vscode.cpptools-extension-pack"
+				"ms-vscode.cpptools-extension-pack",
+				"ms-python.python",
+				"ms-toolsai.jupyter",
+				"tamasfe.even-better-toml",
 			]
 		}
-	}
+	},
+	"mounts": [
+        "source=${localEnv:HOME}/data/machine-learning,target=/data,type=bind,consistency=cached"
+    ]
 }

.gitignore

@@ -14,3 +14,4 @@
 /src/UtilitiesLib/Version.h
 /test-man
 jupyter/.env
+_skbuild/

.vscode/extensions.json

@@ -4,7 +4,9 @@
         "kriegalex.vscode-cudacpp",
         "ms-python.python",
         "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools-extension-pack",
         "ms-vscode.cpptools",
-        "twxs.cmake"
+        "tamasfe.even-better-toml",
+        "mhutchie.git-graph"
     ]
-}
+}

.vscode/settings.json

@@ -1,6 +1,9 @@
 {
-    "C_Cpp.dimInactiveRegions": false,
+    "cmake.configureSettings": {
+        "CMAKE_VERBOSE_MAKEFILE": "OFF"
+    },
+    "cmake.generator": "Ninja",
     "files.associations": {
         "*.cuh": "cpp"
-    }
+    },
 }

CMakeLists.txt

@@ -71,20 +71,20 @@ find_program(CONAN conan)
 if(CONAN)
     if(NOT EXISTS "${CMAKE_BINARY_DIR}/conan.cmake")
        message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan")
-       file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/v0.16.1/conan.cmake"
+       file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/0.18.1/conan.cmake"
                      "${CMAKE_BINARY_DIR}/conan.cmake"
                      TLS_VERIFY ON)
     endif()
 
     include(${CMAKE_BINARY_DIR}/conan.cmake)
-    conan_cmake_run(CONANFILE conanfile.py
-                    BASIC_SETUP CMAKE_TARGETS NO_OUTPUT_DIRS
-                    SETTINGS compiler.libcxx=libstdc++11
-                    BUILD missing)
-
-    include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
-    set(CONAN_SYSTEM_INCLUDES "On")
-    conan_basic_setup(NO_OUTPUT_DIRS)
+
+    conan_cmake_configure(REQUIRES gtest/1.13.0 pybind11/2.10.4
+                          GENERATORS cmake_find_package)
+    conan_cmake_autodetect(settings)
+    conan_cmake_install(PATH_OR_REFERENCE .
+                        BUILD missing
+                        REMOTE conancenter
+                        SETTINGS ${settings})
 endif()
 
 # Get Git revision
@@ -146,7 +146,7 @@ find_package(pybind11)
 add_subdirectory(src)
 
 find_package(GTest)
-if(GTEST_FOUND)
+if(GTEST_FOUND AND (NOT DEFINED SKBUILD))
     ENABLE_TESTING()
     add_subdirectory(test)
 endif()
@@ -163,16 +163,20 @@ if(DOXYGEN_FOUND)
     )
 endif()
 
-install(DIRECTORY scripts/
-    DESTINATION scripts
-    FILES_MATCHING PATTERN "*.py"
-    PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
-                GROUP_EXECUTE GROUP_READ
-                WORLD_EXECUTE WORLD_READ
-)
+if(DEFINED SKBUILD)
+    install(FILES python/pink/tools.py python/pink/__init__.py DESTINATION pink)
+else()
+    install(DIRECTORY python/
+        DESTINATION python
+        FILES_MATCHING PATTERN "*.py"
+        PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
+                    GROUP_EXECUTE GROUP_READ
+                    WORLD_EXECUTE WORLD_READ
+    )
+endif()
 
 if(UNIX)
-    set(CPACK_CMAKE_GENERATOR "Unix Makefiles")
+    set(CPACK_CMAKE_GENERATOR "Ninja")
     set(CPACK_SOURCE_GENERATOR "TGZ;TBZ2")
     set(CPACK_GENERATOR "TGZ;TBZ2;DEB;RPM")
     set(CPACK_PACKAGE_MAINTAINER "Bernd Doser")

MANIFEST.in

@@ -0,0 +1,57 @@
+# For developers
+
+## Create PyPI package
+
+```
+python3 -m build
+
+python3 -m wheel unpack dist/astro_pink-2.5-cp310-cp310-manylinux_2_35_x86_64.whl 
+cd astro_pink-2.5/
+patchelf --set-rpath '$ORIGIN/lib' pink.cpython-310-x86_64-linux-gnu.so 
+ldd pink.cpython-310-x86_64-linux-gnu.so 
+cd ..
+python3 -m wheel pack astro_pink-2.5
+mv astro_pink-2.5-cp310-cp310-manylinux_2_35_x86_64.whl dist/
+
+python3 -m twine upload --repository [pypi|testpypi] dist/*
+```
+
+### Overwrite deployed wheels
+
+It is not allowed to upload the same filename twice to testpypi or pypi.
+Therefore, build numbers can be used:
+
+```
+astro_pink-2.5-cp310-cp310-manylinux_2_35_x86_64.whl
+astro_pink-2.5-1-cp310-cp310-manylinux_2_35_x86_64.whl
+astro_pink-2.5-2-cp310-cp310-manylinux_2_35_x86_64.whl
+```
+
+
+## Manylinux container
+
+docker run -it -v $PWD:/work -w /work bernddoser/manylinux2010-cuda /bin/bash
+
+/opt/python/cp37-cp37m/bin/pip install -U auditwheel
+
+/opt/python/<python version>/bin/pip wheel -v . -w output
+
+
+
+## Useful commands for debugging
+
+```
+auditwheel repair astro_pink-2.4.1-cp36-cp36m-linux_x86_64.whl 
+auditwheel show wheelhouse/astro_pink-2.4.1-cp36-cp36m-manylinux2010_x86_64.whl 
+```
+```
+ldd pink.cpython-36m-x86_64-linux-gnu.so 
+readelf -d pink.cpython-36m-x86_64-linux-gnu.so
+```
+```
+/opt/python/cp37-cp37m/bin/wheel unpack astro_pink-2.4.1-cp36-cp36m-manylinux2010_x86_64.whl 
+cd astro_pink-2.4.1
+mv libCudaLib.so libPythonBindingLib.so astro_pink.libs
+cd ..
+/opt/python/cp37-cp37m/bin/wheel pack astro_pink-2.4.1
+```

README-dev.md

@@ -2,6 +2,8 @@
 [![Join the chat at https://gitter.im/HITS-AIN-PINK/Lobby](https://badges.gitter.im/HITS-AIN-PINK/Lobby.svg)](https://gitter.im/HITS-AIN-PINK/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 <a href="http://ascl.net/1910.001"><img src="https://img.shields.io/badge/ascl-1910.001-blue.svg?colorB=262255" alt="ascl:1910.001" /></a>
 [![PyPI version](https://badge.fury.io/py/astro-pink.svg)](https://badge.fury.io/py/astro-pink)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/BerndDoser/PINK/blob/pypi/colab/pink-train-demo.ipynb)
+
 
 # Parallelized rotation and flipping INvariant Kohonen maps (PINK)
 
@@ -73,30 +75,6 @@ For conversion and visualization of images and SOM some python scripts are avail
   - train.py:                       SOM training using the PINK Python interface
 
 
-## Benchmarks
-
-The input data for the SOM training are radio-synthesis images of Radio Galaxy Zoo containing 176750 images of the dimension 124x124.
-The SOM layout is hexagonal of the dimension 21x21 which has 331 neurons (see image above). The size of the neurons is 64x64.
-The accuracy for the rotational invariance is 1 degree and the flip invariance is used.
-
-|                                   | PINK 1 | Pink 2 |
-| :---                              |   ---: |   ---: |
-| CPU-1                             |        |  35373 |
-| CPU-1 +    NVIDIA Tesla P40       |   3069 |    909 |
-| CPU-1 + 2x NVIDIA Tesla P40       |   2069 |    636 |
-| CPU-1 + 4x NVIDIA Tesla P40       |   1891 |    858 |
-| CPU-2 +    NVIDIA RTX 2080        |        |    673 |
-| CPU-3 +    NVIDIA GTX 750 Ti      |        |   7185 |
-| CPU-4 + 2x NVIDIA RTX 2080 SUPER  |        |    477 |
-
-All times are in seconds.
-
-  - CPU-1: Intel Gold 5118 (2 sockets, 12 physical cores per socket)
-  - CPU-2: Intel Core i7-8700K (1 socket, 6 physical cores per socket)
-  - CPU-3: Intel Core i7-4790K (1 socket, 4 physical cores per socket)
-  - CPU-4: Intel Gold 6230 (1 socket, 20 physical cores per socket)
-
-
 ## Publication
 
 [Kai Lars Polsterer](https://github.com/kai-polsterer), Fabian Gieseke, Christian Igel,

README.md

@@ -0,0 +1,55 @@
+# Benchmark
+
+## Demo Shapes
+
+Data dim: (4000, 64, 64)
+SOM dim: (8, 8)
+Rotations: 360
+GPU: RTX 2080
+
+| Execution                                           | s/it |
+|:---------                                           |  ---:|
+| binary with CUDA [^1]                               |    7 |
+| binary without CUDA [^2]                            |  135 |
+| train.py [^3]                                       |  127 |
+| colab demo without CUDA                             |  127 |
+| colab demo without CUDA @ colab.research.google.com |  170 |
+
+[^1]
+```
+./build/bin/Pink --train /data/pink/shapes_v2.bin som.bin --som-width 8 --som-height 8
+```
+
+[^2] `--cuda-off`
+
+[^3]
+```
+./python/pink/train.py /data/pink/shapes/* -v
+```
+
+
+## Radio Galaxy Zoo
+
+Data (176750, 124, 124)
+SOM (21, 21)
+
+The input data for the SOM training are radio-synthesis images of Radio Galaxy Zoo containing 176750 images of the dimension 124x124.
+The SOM layout is hexagonal of the dimension 21x21 which has 331 neurons (see image above). The size of the neurons is 64x64.
+The accuracy for the rotational invariance is 1 degree and the flip invariance is used.
+
+|                                   | PINK 1 | PINK 2 |
+| :---                              |   ---: |   ---: |
+| CPU-1                             |        |  35373 |
+| CPU-1 +    NVIDIA Tesla P40       |   3069 |    909 |
+| CPU-1 + 2x NVIDIA Tesla P40       |   2069 |    636 |
+| CPU-1 + 4x NVIDIA Tesla P40       |   1891 |    858 |
+| CPU-2 +    NVIDIA RTX 2080        |        |    673 |
+| CPU-3 +    NVIDIA GTX 750 Ti      |        |   7185 |
+| CPU-4 + 2x NVIDIA RTX 2080 SUPER  |        |    477 |
+
+All times are in seconds.
+
+  - CPU-1: Intel Gold 5118 (2 sockets, 12 physical cores per socket)
+  - CPU-2: Intel Core i7-8700K (1 socket, 6 physical cores per socket)
+  - CPU-3: Intel Core i7-4790K (1 socket, 4 physical cores per socket)
+  - CPU-4: Intel Gold 6230 (1 socket, 20 physical cores per socket)