Added a setup.py, auto-detect subdir/arch, allow insecure server (for debug, or easy proxy behind nginx), update README

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2018, SciTools-incubator
+Copyright (c) 2018, Met Office.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.md

@@ -9,7 +9,7 @@ To use, simply clone this repo and run the Python webserver, observing the
 ## Usage
 
 ```
-usage: artifact_upload_handler.py [-h] -d WRITE_DIR -p PORT -e CONDA_EXE -c
+usage: conda-upload-server [-h] -d WRITE_DIR -p PORT -e CONDA_EXE -c
                                   CERTFILE -k KEYFILE -t TOKEN_HASH
 
 optional arguments:
@@ -27,23 +27,52 @@ optional arguments:
                         hash of secure token
 ```
 
+### Basic Usage Example
 
-### Usage Example 
+To start an insecure webserver that will write packages to a conda channel in the
+given directory:
+
+```
+$ conda-upload-server -d /path/to/channel
+```
+
+To test the webserver:
+
+```
+$ curl -F 'artifact=@/path/to/conda/binary/package.tar.bz2' --fail http://localhost:8080/
+```
+
+This will create a conda channel in /path/to/channel/<package-platform>/, with the
+appropriate channel (``repodata.json``) content.
 
-For example, to start a secure webserver running on port 9999 that will write
-linux-64 packages to the (imaginary) conda channel at ``/path/to/conda/channel``:
+
+### Secure Usage Example
+
+First, we need to generate the token that must be kept secret on the server.
+We start with a secret held by the client, and optionally a salt kept by the server:
 
 ```
-$ python artifact_upload_handler.py -d /path/to/conda/channel/linux-64 \
-                                    -p 9999 \
-                                    -e /path/to/env/bin/conda \
-                                    -c /path/to/mycertfile.crt \
-                                    -k /path/to/mykeyfile.key \
-                                    -t eccd989b
+$ python -m conda_upload_server.token the_client_secret_token
+
+Server salt:  "42679ad04d44c96ed27470c02bfb28c3"
+Client token: "the_client_secret_token"
+Server token: "72a61a0d67edd573649354adc1fce4b7e1f56add1d03ddc328fa032060c8373f"
 
 ```
 
-To test the webserver:
+Now we start a secure webserver running on port 9999 that will write
+linux-64 packages to the conda channel at ``/path/to/conda/channel/linux-64``:
+
+```
+$ conda-upload-server -d /path/to/conda/channel/ \
+                       -p 9999 \
+                       -c /path/to/mycertfile.crt \
+                       -k /path/to/mykeyfile.key \
+                       -t 72a61a0d67edd573649354adc1fce4b7e1f56add1d03ddc328fa032060c8373f
+
+```
+
+To test the webserver using python and requests:
 
 ```python
 import requests
@@ -53,7 +82,7 @@ artifacts = [open('my-artifact1-1.0.0-2.tar.bz2', 'rb'),
              open('my-artifact3-0.9.1-0.tar.bz2', 'rb'),
             ]
 url = 'https://localhost:9999/'
-token = 'mysecuretoken'
+token = 'the_client_secret_token'
 
 for artifact in artifacts:
     requests.post(url, data={'token': token}, files={'artifact': artifact},

conda_upload_server/cli.py

@@ -0,0 +1,5 @@
+from . import handler
+
+
+def main():
+    return handler.main()

handler/artifact_upload_handler.py → conda_upload_server/handler.py

@@ -5,20 +5,32 @@
 """
 
 import argparse
+import distutils.spawn
 import hashlib
 import hmac
+import io
+import json
+import logging
 import os
 import subprocess
+import tarfile
 
 import tornado.httpserver
 import tornado.ioloop
+from tornado.log import enable_pretty_logging
 import tornado.web
+from tornado.web import RequestHandler, Finish
+
+from .token import DEFAULT_SALT, check_token as _check_token
+
+
+logger = logging.getLogger(__name__)
 
 
 __version__ = '0.1.0dev0'
 
 
-class ArtifactUploadHandler(tornado.web.RequestHandler):
+class ArtifactUploadHandler(RequestHandler):
     """Handle writing conda build artifacts to a channel."""
 
     def initialize(self, write_path, conda_exe, hash_expected):
@@ -37,41 +49,57 @@ def initialize(self, write_path, conda_exe, hash_expected):
         self.conda_exe = conda_exe
         self.hash_expected = hash_expected
 
-    def _check_token(self, token):
-        """
-        Check we have been passed the *correct* secure token before uploading
-        the artifact to the channel.
-
-        """
-        salt = '42679ad04d44c96ed27470c02bfb28c3'
-        to_hash = '{}{}'.format(salt, token)
-        hash_result = hashlib.sha256(to_hash).hexdigest()
-        # Reduce the risk of timing analysis attacks.
-        return hmac.compare_digest(self.hash_expected, hash_result)
-
     def _conda_index(self, directory):
         """
         Update the package index metadata files in the provided directory.
 
         """
         cmds = [self.conda_exe, 'index', directory]
-        subprocess.check_call(cmds)
+        output = subprocess.check_output(cmds)
+        logger.info('\n' + output.decode('utf-8').strip())
 
     def post(self):
-        token = self.get_argument('token')
-        arch = self.get_argument('arch', default='none')
-        arch = '' if arch == 'none' else arch
+        token = self.get_argument('token', default=None)
         file_data, = self.request.files['artifact']
         filename = file_data['filename']
         body = file_data['body']
-        if self._check_token(token):
-            directory = os.path.join(self.write_path, arch)
-            target = os.path.join(directory, filename)
-            with open(target, 'wb') as owfh:
-                owfh.write(body)
-            self._conda_index(directory)
-        else:
-            self.send_error(401)
+
+        if not _check_token(self.hash_expected, DEFAULT_SALT, token):
+            self.set_status(401)
+            logger.info('Unauthorized token request')
+            raise Finish()
+
+        f = io.BytesIO(body)
+        subdir = subdir_of_binary(f)
+
+        directory = os.path.join(self.write_path, subdir)
+        if not os.path.exists(directory):
+            logger.info('Creating a new channel at {}'.format(directory))
+            os.makedirs(directory)
+        target = os.path.join(directory, filename)
+
+        # Don't overwrite existing binaries.
+        if os.path.exists(target):
+            self.set_status(401)
+            logger.info('Attempting to overwrite an existing binary')
+            raise Finish()
+
+        with open(target, 'wb') as owfh:
+            owfh.write(body)
+        self._conda_index(directory)
+
+
+def subdir_of_binary(fh):
+    """Given a file handle to a tar.bz2 conda binary, identify the arch"""
+    subdir = None
+    with tarfile.open(fileobj=fh, mode="r:bz2") as tar:
+        fh = tar.extractfile('info/index.json')
+        if fh is not None:
+            index = json.loads(fh.read().decode('utf-8'))
+            subdir = index.get('subdir', None)
+    if not subdir:
+        raise ValueError('Subdir cannot be determined for binary')
+    return subdir
 
 
 def make_app(write_path, conda_exe, token_hash):
@@ -81,32 +109,33 @@ def make_app(write_path, conda_exe, token_hash):
     return tornado.web.Application([(r'/', ArtifactUploadHandler, kw)])
 
 
-if __name__ == '__main__':
+def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-d", "--write_dir",
                         help="directory to write artifacts to",
-                        required=True,
+                        default=os.getcwd(),
                         )
     parser.add_argument("-p", "--port",
                         help="webserver port number",
-                        required=True,
+                        default=8080,
                         )
     parser.add_argument("-e", "--conda_exe",
                         help="full path to conda executable",
-                        required=True,
+                        default=None,
                         )
     parser.add_argument("-c", "--certfile",
                         help="full path to certificate file for SSL",
-                        required=True,
+                        default=None,
                         )
     parser.add_argument("-k", "--keyfile",
                         help="full path to keyfile for SSL",
-                        required=True,
+                        default=None,
                         )
     parser.add_argument("-t", "--token_hash",
                         help="hash of secure token",
-                        required=True,
+                        default=None,
                         )
+
     args = parser.parse_args()
     write_path = args.write_dir
     port = args.port
@@ -115,9 +144,30 @@ def make_app(write_path, conda_exe, token_hash):
     keyfile = args.keyfile
     token_hash = args.token_hash
 
-    ssl_opts = {'certfile': certfile,
-                'keyfile': keyfile}
+
+    if not conda_exe:
+        conda_exe = distutils.spawn.find_executable("conda")
+
+    if certfile or keyfile:
+        ssl_opts = {'certfile': certfile,
+                    'keyfile': keyfile}
+    else:
+        ssl_opts = None
+
+    enable_pretty_logging()
+
+    url = '{protocol}{host}:{port}'.format(
+        protocol='https://' if ssl_opts else 'http://',
+        host='localhost',
+        port=port)
+
+    logger.info('Serving on {url}'.format(url=url))
+
     app = make_app(write_path, conda_exe, token_hash)
-    https_server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts)
-    https_server.listen(port)
+    server = tornado.httpserver.HTTPServer(app, ssl_options=ssl_opts)
+    server.listen(port)
     tornado.ioloop.IOLoop.current().start()
+
+
+if __name__ == '__main__':
+    main()

conda_upload_server/token.py

@@ -0,0 +1,57 @@
+import argparse
+import distutils.spawn
+import hashlib
+import hmac
+import io
+import json
+import logging
+import os
+import subprocess
+import tarfile
+
+
+#: A default salt is provided, but it is recommended to use your own.
+DEFAULT_SALT = '42679ad04d44c96ed27470c02bfb28c3'
+
+
+def generate_hash(token, *, salt=DEFAULT_SALT):
+    to_hash = '{}{}'.format(salt, token)
+    return hashlib.sha256(to_hash.encode('utf-8')).hexdigest()
+
+
+def check_token(hash_expected, salt, token):
+    """Check we have been passed the correct secure token."""
+    if hash_expected:
+        hash_result = generate_hash(salt=salt, token=token)
+        # Reduce the risk of timing analysis attacks.
+        result = hmac.compare_digest(hash_expected, hash_result)
+    else:
+        assert token is None
+        result = True
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--salt",
+                        help="The (semi-)secret salt being used by the server",
+                        default=DEFAULT_SALT,
+                        )
+    parser.add_argument("token",
+                        help="The secret token/password made available to the client in plain-text",
+                        )
+
+    args = parser.parse_args()
+
+    server_token = generate_hash(salt=args.salt, token=args.token)
+
+    print('Server salt:  "{}"'.format(args.salt))
+    print('Client token: "{}"'.format(args.token))
+    print('Server token: "{}"'.format(server_token))
+
+    assert check_token(server_token, args.salt, args.token)
+
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,43 @@
+from codecs import open
+from os import path
+from setuptools import setup, find_packages
+
+
+here = path.abspath(path.dirname(__file__))
+
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+
+setup(
+    name='conda-upload-server',
+    version='1.0',
+    description='A lightweight webservice capable of updating a local conda channel',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://github.com/SciTools-incubator/artifact-upload-handler',
+
+    author='SciTools contributors',
+    author_email='[email protected]',
+
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+    ],
+
+    keywords='conda upload channel',
+    packages=find_packages(),
+
+    python_requires='>=3.5',
+    install_requires=[
+        'tornado'],
+
+    entry_points={  # Optional
+        'console_scripts': [
+            'conda-upload-server=conda_upload_server.cli:main',
+        ],
+    },
+
+    project_urls={
+        'Source': 'https://github.com/SciTools-incubator/artifact-upload-handler',
+    },
+)