Merge branch 'develop'

.travis.yml

@@ -5,5 +5,5 @@ python:
 install: "pip install -r requirements.txt"
 # command to run tests
 script:
-    - nosetests -w ./tests -vv
+    - nosetests -w snare/tests -vv
     - pycodestyle . --max-line-length=120

bin/clone

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+"""
+Copyright (C) 2015-2016 MushMush Foundation
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+"""
+
+import argparse
+import asyncio
+import os
+import sys
+from snare.utils import logger
+from snare.cloner import Cloner
+from snare.utils.snare_helpers import str_to_bool
+
+def main():
+    if os.getuid() != 0:
+        print('Clone has to be run as root!')
+        sys.exit(1)
+    if not os.path.exists('/opt/snare'):
+        os.mkdir('/opt/snare')
+    if not os.path.exists('/opt/snare/pages'):
+        os.mkdir('/opt/snare/pages')
+    loop = asyncio.get_event_loop()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target", help="domain of the site to be cloned", required=True)
+    parser.add_argument("--max-depth", help="max depth of the cloning", required=False, default=sys.maxsize)
+    parser.add_argument("--log_path", help="path to the error log file")
+    parser.add_argument(
+        "--css-validate", help="set whether css validation is required", type=str_to_bool, default=None
+    )
+    args = parser.parse_args()
+    if args.log_path:
+        log_err = args.log_path + "clone.err"
+    else:
+        log_err = "/opt/snare/clone.err"
+    logger.Logger.create_clone_logger(log_err, __package__)
+    print("Error logs will be stored in {}\n".format(log_err))
+    try:
+        cloner = Cloner(args.target, int(args.max_depth), args.css_validate)
+        loop.run_until_complete(cloner.get_root_host())
+        loop.run_until_complete(cloner.run())
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == '__main__':
+    print("""
+    ______ __      ______ _   ____________
+   / ____// /     / __  // | / / ____/ __ \\
+  / /    / /     / / / //  |/ / __/ / /_/ /
+ / /___ / /____ / /_/ // /|  / /___/ _, _/
+/_____//______//_____//_/ |_/_____/_/ |_|
+    
+    """)
+    main()

bin/snare

@@ -0,0 +1,209 @@
+#!/usr/bin/python3
+
+"""
+Copyright (C) 2015-2016 MushMush Foundation
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+"""
+import argparse
+import asyncio
+import pwd
+import grp
+import configparser
+import json
+import multiprocessing
+import os
+import sys
+import time
+import uuid
+from concurrent.futures import ProcessPoolExecutor
+import aiohttp
+import git
+import pip
+import netifaces as ni
+from snare.server import HttpRequestHandler
+from snare.utils.logger import Logger
+from snare.utils import snare_helpers
+from snare.utils.snare_helpers import str_to_bool
+
+
+def create_initial_config():
+    cfg = configparser.ConfigParser()
+    cfg['WEB-TOOLS'] = dict(google='', bing='')
+    with open('/opt/snare/snare.cfg', 'w') as configfile:
+        cfg.write(configfile)
+
+
+def snare_setup():
+    if os.getuid() != 0:
+        print('Snare has to be started as root!')
+        sys.exit(1)
+    # Create folders
+    if not os.path.exists('/opt/snare'):
+        os.mkdir('/opt/snare')
+    if not os.path.exists('/opt/snare/pages'):
+        os.mkdir('/opt/snare/pages')
+    # Write pid to pid file
+    with open('/opt/snare/snare.pid', 'wb') as pid_fh:
+        pid_fh.write(str(os.getpid()).encode('utf-8'))
+    # Config file
+    if not os.path.exists('/opt/snare/snare.cfg'):
+        create_initial_config()
+    # Read or create the sensor id
+    uuid_file_path = '/opt/snare/snare.uuid'
+    if os.path.exists(uuid_file_path):
+        with open(uuid_file_path, 'rb') as uuid_fh:
+            snare_uuid = uuid_fh.read()
+        return snare_uuid
+    else:
+        with open(uuid_file_path, 'wb') as uuid_fh:
+            snare_uuid = str(uuid.uuid4()).encode('utf-8')
+            uuid_fh.write(snare_uuid)
+        return snare_uuid
+
+
+def drop_privileges():
+    uid_name = 'nobody'
+    wanted_user = pwd.getpwnam(uid_name)
+    gid_name = grp.getgrgid(wanted_user.pw_gid).gr_name
+    wanted_group = grp.getgrnam(gid_name)
+    os.setgid(wanted_group.gr_gid)
+    os.setuid(wanted_user.pw_uid)
+    new_user = pwd.getpwuid(os.getuid())
+    new_group = grp.getgrgid(os.getgid())
+    print('privileges dropped, running as "{}:{}"'.format(new_user.pw_name, new_group.gr_name))
+
+
+def compare_version_info(timeout):
+    while True:
+        repo = git.Repo(os.getcwd())
+        try:
+            rem = repo.remote()
+            res = rem.fetch()
+            diff_list = res[0].commit.diff(repo.heads.master)
+        except TimeoutError:
+            print('timeout fetching the repository version')
+        else:
+            if diff_list:
+                print('you are running an outdated version, SNARE will be updated and restarted')
+                repo.git.reset('--hard')
+                repo.heads.master.checkout()
+                repo.git.clean('-xdf')
+                repo.remotes.origin.pull()
+                pip.main(['install', '-r', 'requirements.txt'])
+                os.execv(sys.executable, [sys.executable, __file__] + sys.argv[1:])
+                return
+            else:
+                print('you are running the latest version')
+            time.sleep(timeout)
+
+
+async def check_tanner():
+    vm = snare_helpers.VersionManager()
+    async with aiohttp.ClientSession() as client:
+        req_url = 'http://{}:8090/version'.format(args.tanner)
+        try:
+            resp = await client.get(req_url)
+            result = await resp.json()
+            version = result["version"]
+            vm.check_compatibility(version)
+        except aiohttp.ClientOSError:
+            print("Can't connect to tanner host {}".format(req_url))
+            exit(1)
+        else:
+            await resp.release()
+
+if __name__ == '__main__':
+    print(r"""
+   _____ _   _____    ____  ______
+  / ___// | / /   |  / __ \/ ____/
+  \__ \/  |/ / /| | / /_/ / __/
+ ___/ / /|  / ___ |/ _, _/ /___
+/____/_/ |_/_/  |_/_/ |_/_____/
+
+    """)
+    parser = argparse.ArgumentParser()
+    page_group = parser.add_mutually_exclusive_group(required=True)
+    page_group.add_argument("--page-dir", help="name of the folder to be served")
+    page_group.add_argument("--list-pages", help="list available pages", action='store_true')
+    parser.add_argument("--index-page", help="file name of the index page", default='index.html')
+    parser.add_argument("--port", help="port to listen on", default='8080')
+    parser.add_argument("--interface", help="interface to bind to")
+    parser.add_argument("--host-ip", help="host ip to bind to", default='localhost')
+    parser.add_argument("--debug", help="run web server in debug mode", default=False)
+    parser.add_argument("--tanner", help="ip of the tanner service", default='tanner.mushmush.org')
+    parser.add_argument("--skip-check-version", help="skip check for update", action='store_true')
+    parser.add_argument("--slurp-enabled", help="enable nsq logging", action='store_true')
+    parser.add_argument("--slurp-host", help="nsq logging host", default='slurp.mushmush.org')
+    parser.add_argument("--slurp-auth", help="nsq logging auth", default='slurp')
+    parser.add_argument("--config", help="snare config file", default='snare.cfg')
+    parser.add_argument("--auto-update", help="auto update SNARE if new version available ", default=True)
+    parser.add_argument("--update-timeout", help="update snare every timeout ", default='24H')
+    parser.add_argument("--server-header", help="set server-header", default='nignx/1.3.8')
+    parser.add_argument("--no-dorks", help="disable the use of dorks", type=str_to_bool,  default=True)
+    parser.add_argument("--log-dir", help="path to directory of the log file", default='/opt/snare/')
+    args = parser.parse_args()
+    base_path = '/opt/snare/'
+    base_page_path = '/opt/snare/pages/'
+    snare_uuid = snare_setup()
+    config = configparser.ConfigParser()
+    config.read(os.path.join(base_path, args.config))
+    log_debug = args.log_dir + "snare.log"
+    log_err = args.log_dir + "snare.err"
+    Logger.create_logger(log_debug, log_err, __package__)
+    if args.list_pages:
+        print('Available pages:\n')
+        for page in os.listdir(base_page_path):
+            print('\t- {}'.format(page))
+        print('\nuse with --page-dir {page_name}\n\n')
+        exit()
+    full_page_path = os.path.join(base_page_path, args.page_dir)
+    if not os.path.exists(full_page_path):
+        print("--page-dir: {0} does not exist".format(args.page_dir))
+        exit()
+    args.index_page = os.path.join("/", args.index_page)
+
+    if not os.path.exists(os.path.join(full_page_path, 'meta.json')):
+        conv = snare_helpers.Converter()
+        conv.convert(full_page_path)
+        print("pages was converted. Try to clone again for the better result.")
+
+    with open(os.path.join(full_page_path, 'meta.json')) as meta:
+        meta_info = json.load(meta)
+    if not os.path.exists(os.path.join(base_page_path, args.page_dir,
+                                       os.path.join(meta_info[args.index_page]['hash']))):
+        print('can\'t create meta tag')
+    else:
+        snare_helpers.add_meta_tag(args.page_dir, meta_info[args.index_page]['hash'], config)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(check_tanner())
+
+    pool = ProcessPoolExecutor(max_workers=multiprocessing.cpu_count())
+    compare_version_fut = None
+    if args.auto_update is True:
+        timeout = snare_helpers.parse_timeout(args.update_timeout)
+        compare_version_fut = loop.run_in_executor(pool, compare_version_info, timeout)
+
+    if args.host_ip == 'localhost' and args.interface:
+        args.host_ip = ni.ifaddresses(args.interface)[2][0]['addr']
+
+    app = HttpRequestHandler(meta_info, args, snare_uuid, debug=args.debug, keep_alive=75)
+    drop_privileges()
+    print('serving with uuid {0}'.format(snare_uuid.decode('utf-8')))
+    print("Debug logs will be stored in", log_debug)
+    print("Error logs will be stored in", log_err)
+    try:
+        app.start()
+    except (KeyboardInterrupt, TypeError) as e:
+        print(e)
+    finally:
+        if compare_version_fut:
+            compare_version_fut.cancel()

requirements.txt

@@ -1,8 +1,9 @@
-aiohttp<2.0
+aiohttp
+aiohttp_jinja2
+yarl
 beautifulsoup4
 cssutils
 gitpython
 netifaces
-yarl==0.9.8
 python-magic
 pycodestyle

setup.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+from setuptools import find_packages
+from distutils.core import setup
+
+setup(name='Snare',
+      version='0.3.0',
+      description='Super Next generation Advanced Reactive honEypot',
+      author='MushMush Foundation',
+      author_email='[email protected]',
+      url='https://github.com/mushorg/snare',
+      packages=find_packages(exclude=['*.pyc']),
+      scripts=['bin/snare', 'bin/clone'],
+      )