Initial commit

Author: skizzerz

.gitignore

@@ -0,0 +1,3 @@
+venv/
+config.py
+.idea/

LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2019, Ryan Schmidt
+All Rights Reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

app.py

@@ -0,0 +1,79 @@
+from sys import stderr
+import string
+import random
+from flask import Flask, request, redirect, jsonify, abort, render_template
+from pygments import highlight
+from pygments.lexers import Python3TracebackLexer
+from pygments.formatters import HtmlFormatter
+import db
+
+app = Flask(__name__)
+
+
+def _wants_json():
+    """
+    Determine if the user wants json or html output for APIs which can serve both.
+
+    We prefer sending html over json, and only allow json if the Accept header contains application/json
+    and either lacks text/html or has application/json has higher quality than text/html.
+
+    :return: True if the user should be served json, False if the user should be served html
+    """
+    if not request.accept_mimetypes.accept_json:
+        return False  # if json isn't allowed, don't serve it
+    if not request.accept_mimetypes.accept_html:
+        return True  # json is allowed but html isn't => serve json
+    # otherwise prefer to serve html over json, respecting the quality param on Accept header
+    best = request.accept_mimetypes.best_match(("text/html", "application/json"))
+    if best != "application/json":
+        return False  # html is of greater quality than json
+    if request.accept_mimetypes.quality("text/html") == request.accept_mimetypes.quality("application/json"):
+        return False  # html is of equal quality to json, and we prefer html
+    return True  # json is of higher quality than html
+
+
+@app.route("/submit", methods=["POST"])
+def submit():
+    data = request.get_json(silent=True)
+    if not isinstance(data, dict) or not data.get("c", None):
+        return jsonify({"status": "error", "error": "Invalid request"}), 400
+    charset = string.ascii_letters + string.digits
+    with db.DbConnection() as conn:
+        slug = "".join(random.choice(charset) for _ in range(4))
+        while len(slug) < 17:
+            found = len(conn.execute("SELECT paste_slug FROM pastes WHERE paste_slug = %s", (slug,)))
+            if found == 0:
+                break
+            slug += random.choice(charset)
+        if len(slug) == 17:
+            # stderr is logged so we can examine these events later
+            print(f"Unable to get unique URL slug, tried {slug}", file=stderr)
+            return jsonify({"status": "error", "error": "Unable to get unique URL slug"}), 500
+        conn.execute("""INSERT INTO pastes (paste_slug, paste_expires, paste_type, paste_content)
+                     VALUES (%s, DATE_ADD(NOW(), INTERVAL 7 DAY), 'paste', %s)""", (slug, data["c"]))
+        return jsonify({"status": "success", "url": request.url_root + slug})
+
+@app.route("/<slug>")
+def get_paste(slug):
+    if len(slug) > 16:
+        if _wants_json():
+            return jsonify({"status": "error", "error": "Invalid request"}), 400
+        else:
+            abort(400)
+    with db.DbConnection() as conn:
+        data = conn.fetch("SELECT * FROM pastes WHERE paste_slug = %s", (slug,))
+    if not data:
+        if _wants_json():
+            return jsonify({"status": "error", "error": "Paste not found"}), 404
+        else:
+            abort(404)
+    if data["paste_type"] == "redirect":
+        redirect(data["paste_content"])
+    if _wants_json():
+        return jsonify({"status": "success", "data": data["paste_content"], "expires": data["paste_expires"]})
+    return highlight(data["paste_content"],
+                     Python3TracebackLexer(),
+                     HtmlFormatter(full=True, linenos="table", lineanchors="l", anchorlinenos=True, wrapcode=True))
+
+if __name__ == "__main__":
+    app.run()

config.py.example

@@ -0,0 +1,34 @@
+# Session key. Changing this will invalidate all existing admin sessions.
+# This should be random!
+SESSION_KEY = b""
+
+# Database credentials, used to store paste info
+DB_HOST = "localhost"  # Hostname of MySQL server or absolute path to unix socket
+DB_PORT = 3306  # Port of MySQL server (not used if DB_HOST points to a unix socket)
+DB_NAME = ""  # Database name
+DB_USER = "some_username"  # Database user, must have read/write access to db
+DB_PASS = "some_password"  # Password for database user
+
+# Notification (webhook) settings
+# If enabled, POST to this address with a payload indicating a new paste
+# was made. The POST body will be a JSON document detailed in webhooks.txt.
+WEBHOOK_ENABLE = False  # Whether or not to enable the webhook
+WEBHOOK_URL = ""  # URL to POST to
+# PEM encoded private key to sign webhook
+WEBHOOK_KEY = """
+"""
+
+# OAuth (admin area) settings. Admins can view/manage all pastes and create short URL redirects
+OAUTH_ENABLE = False  # Whether to enable the admin area. Requires OAuth login.
+OAUTH_ENDPOINT = ""  # Base URL to wiki, not including index.php or api.php
+OAUTH_CONSUMER_KEY = ""  # OAuth consumer key
+OAUTH_SECRET_KEY = ""  # OAuth secret key
+
+# ElasticSearch settings
+ELASTIC_ENABLE = False  # Whether to send incoming pastes to ElasticSearch
+ELASTIC_URL = ""  # URL to ES instance. Cloud ID also supported here
+ELASTIC_PORT = 0  # Port of ES instance
+ELASTIC_USER = ""  # If ES requires auth, the username to use (optional)
+ELASTIC_PASS = ""  # If ES requires auth, the password to use (optional)
+ELASTIC_INDEX = ""  # Index name to use. The user must have ability to write to this index
+ELASTIC_PIPELINE = ""  # Ingest node pipeline to use (optional)

cron.py

@@ -0,0 +1,10 @@
+import config
+
+
+def run():
+    """Run cron tasks (deleting expired stuff)"""
+    pass
+
+
+if __name__ == "__main__":
+    run()

db.py

@@ -0,0 +1,83 @@
+import mysql.connector
+from typing import Union, Dict, List, Optional, Tuple, Any
+
+import config
+
+
+class DbConnection:
+    def __init__(self):
+        self.conn = None  # type: Optional[mysql.connector.MySQLConnection]
+
+    def __enter__(self):
+        args = {
+            "user": config.DB_USER,
+            "password": config.DB_PASS,
+            "database": config.DB_NAME,
+            "charset": "utf8mb4",
+            "collation": "utf8mb4_unicode_ci"
+        }
+
+        if config.DB_HOST[0] == "/":
+            # using unix socket
+            args["unix_socket"] = config.DB_HOST
+        else:
+            args["host"] = config.DB_HOST
+            args["port"] = config.DB_PORT
+
+        self.conn = mysql.connector.connect(**args)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Close the db connection and commit/rollback any pending transactions.
+
+        If an exception was raised, we rollback. Otherwise, we commit.
+        """
+        if self.conn and self.conn.is_connected():
+            try:
+                if exc_type is None:
+                    self.conn.commit()
+                else:
+                    self.conn.rollback()
+            finally:
+                self.conn.close()
+                self.conn = None  # type: Optional[mysql.connector.MySQLConnection]
+
+    def execute(self, operation: Any, params: Tuple = (), multi: bool = False)\
+            -> Union[int, List[Union[int, Dict[str, Any], List[Dict[str, Any]]]]]:
+        c = self.conn.cursor(dictionary=True)
+        try:
+            result = c.execute(operation, params=params, multi=multi)
+            if multi:
+                data = []
+                for r in result:
+                    if r.with_rows:
+                        data.append(r.fetchall())
+                    else:
+                        data.append(r.rowcount)
+            else:
+                if c.with_rows:
+                    data = c.fetchall()
+                else:
+                    data = c.rowcount
+        finally:
+            c.close()
+        return data
+
+    def fetch(self, operation: Any, params: Tuple = ()) -> Optional[Dict[str, Any]]:
+        """
+        Fetches the first row of the result set, or None if there are no results.
+
+        Returns None if the operation is not a SELECT query as well. Use execute if you
+        wish to retrieve the number of affected rows for a DML query.
+
+        :param operation: SQL query
+        :param params: Parameters for the query
+        :return: The first row or None
+        """
+        data = self.execute(operation, params)
+        if isinstance(data, list):
+            try:
+                return data[0]
+            except IndexError:
+                return None  # no results
+        return None  # not a query that returned data, so return None

install.py

@@ -0,0 +1,49 @@
+from typing import Set
+import pathlib
+from db import DbConnection
+
+
+def run():
+    """
+    Execute the installer.
+
+    The installer will get the database schema up to current as well as set up any necessary
+    ancillary structures such as ElasticSearch indices and pipelines.
+
+    :return:
+    """
+    with DbConnection() as conn:
+        conn.execute("""CREATE TABLE IF NOT EXISTS migrations (
+            migration_key varchar(150) PRIMARY KEY,
+            migration_time datetime)""")
+        applied_migrations = set()
+        for row in conn.execute("SELECT migration_key FROM migrations"):
+            applied_migrations.add(row["migration_key"])
+        for p in sorted(pathlib.Path("migrations").iterdir()):
+            if p.is_file() and p[-4:] == ".sql":
+                _apply_migration(p.name[:-4], str(p.absolute()), conn, applied_migrations)
+
+
+def _apply_migration(name: str, path: str, conn: DbConnection, already_applied: Set[str]) -> None:
+    """
+    Check if a migration is applied and if not, run it.
+
+    :param name: Migration name (key). This must be unique.
+    :param path: Path to migration sql file.
+    :param conn: Already-open database connection.
+    :param already_applied: A set of which migrations have already been applied. The current migration will be
+        added to this set if successful.
+    :return:
+    """
+    if name in already_applied:
+        return
+
+    with open(path, "r") as f:
+        conn.execute(f.read(), multi=True)
+        already_applied.add(name)
+        conn.execute("INSERT INTO migrations (migration_key, migration_time) VALUES (%s, NOW())",
+                     (name,))
+
+
+if __name__ == "__main__":
+    run()

migrations/0001_initial_schema.sql

@@ -0,0 +1,11 @@
+CREATE TABLE pastes (
+    paste_id int PRIMARY KEY AUTO_INCREMENT,
+    paste_slug varchar(16) NOT NULL COMMENT 'URL slug for the paste, must be unique',
+    paste_expires datetime COMMENT 'When the paste expires, or NULL if it never expires',
+    paste_type enum('paste', 'redirect') NOT NULL
+        COMMENT 'What type of paste this is. If a paste, paste_content refers to the paste value itself. '
+                'If a redirect, paste_content is a URL to redirect the user to.',
+    paste_content text NOT NULL,
+    CONSTRAINT pastes_paste_slug UNIQUE KEY (paste_slug),
+    INDEX pastes_paste_expires (paste_expires)
+);

requirements.txt

@@ -0,0 +1,27 @@
+asn1crypto==0.24.0
+certifi==2019.3.9
+cffi==1.12.3
+chardet==3.0.4
+Click==7.0
+cryptography==2.6.1
+elasticsearch==7.0.2
+Flask==1.0.3
+flask-mwoauth==0.3.70
+future==0.17.1
+idna==2.8
+itsdangerous==1.1.0
+Jinja2==2.10.1
+MarkupSafe==1.1.1
+mwoauth==0.3.3
+mysql-connector-python==8.0.16
+oauthlib==3.0.1
+protobuf==3.7.1
+pycparser==2.19
+Pygments==2.4.2
+PyJWT==1.7.1
+python-dateutil==2.8.0
+requests==2.22.0
+requests-oauthlib==1.2.0
+six==1.12.0
+urllib3==1.25.3
+Werkzeug==0.15.4