app.py

import datetime
import os
import uuid

import json
import flask
import flask_moment
import sqlite3

from jinja2 import Template

DATASETS = os.getenv("DATASETS", "data")
DBFILE = os.getenv("DBFILE", "data/downloads.sqlite")
EXTRA_TIME = os.getenv("EXTRA_TIME", "+1 day")
PASSWORD = os.getenv("PASSWORD", "secret")
READ_SIZE = 4096

app = flask.Flask(__name__)
moment = flask_moment.Moment(app)
connect = sqlite3.connect(DBFILE)
connect.execute(
    f'CREATE TABLE IF NOT EXISTS DOWNLOADS (\
      id TEXT, \
      dataset TEXT, \
      title TEXT, \
      name TEXT, \
      email TEXT, \
      affiliation TEXT, \
      created DATE DEFAULT (datetime(CURRENT_TIMESTAMP)), \
      expires DATE DEFAULT (datetime(CURRENT_TIMESTAMP, \'{EXTRA_TIME}\')), \
      CONSTRAINT id_pk PRIMARY KEY (id))')


def load_data():
    with open(f"{DATASETS}/datasets.json", "r") as fp:
        return json.load(fp)
    return {}


@app.get("/")
def home():
    data = load_data()
    kwargs = {
        "title": "AIFARMS Data Portal",
        "short_description": "Data gathered as part of AIFARMS that is publicly available.",
        "authors": "",
        "data": data
    }
    return flask.render_template("index.html", **kwargs)

def render_template(template, dataset):
    data = load_data()
    if dataset not in data:
        return flask.redirect('/')
    
    kwargs = data[dataset]
    kwargs["dataset"] = dataset
    
    # Check for license.txt file
    dataset_folder = os.path.join(DATASETS, dataset)
    license_file = os.path.join(dataset_folder, "license.txt")
    if os.path.exists(license_file):
        kwargs["license"] = True
    else:
        kwargs["license"] = False
    # Check for datasheet.pdf file
    datasheet_file = os.path.join(dataset_folder, "datasheet.pdf")
    if os.path.exists(datasheet_file):
        kwargs["datasheet"] = True
    else:
        kwargs["datasheet"] = False
    # Check for dataset.zip file
    zipfile = os.path.join(dataset_folder, "dataset.zip")
    if os.path.exists(zipfile):
        kwargs["filesize"] = sizeof_fmt(os.stat(zipfile).st_size)
        kwargs["download_url"] = f"https://data.aifarms.org/download/{dataset}"
    else:
        kwargs["filesize"] = "N/A"
        kwargs["download_url"] = ""

    # Compute download_url based on presence of urls['download'] or urls['github']
    urls = kwargs.get("urls", {})
    if not kwargs["download_url"]:
        if "download" in urls and urls["download"]:
            kwargs["download_url"] = urls["download"]

    keywords = set(["AIFARMS"])
    keywords.update(kwargs.get("keywords", ""))
    kwargs["aifarms_keywords"] = keywords
    
    if template == "view.html":
        kwargs["title"] = f"{kwargs['title']} - AIFARMS Data Portal"
        if isinstance(kwargs.get("contact"), str):
            if "<" in kwargs["contact"]:
                name, email = kwargs["contact"].split("<")
                kwargs["contact"] = {"name": name.strip(), "email": email.strip(" >")}
            else:
                kwargs["contact"] = {"name": kwargs["contact"].strip(), "email": ""}
    
    return flask.render_template(template, **kwargs)


@app.get("/view/<dataset>")
def view_dataset(dataset):
    return render_template("view.html", dataset)


@app.get("/croissant/<dataset>")
def croissant_dataset(dataset):
    return render_template("croissant.json", dataset=dataset)


@app.get("/citation/<dataset>")
@app.get("/cff/<dataset>")
def citation_dataset(dataset):
    return render_template("citation.cff", dataset=dataset)

@app.get("/sitemap.xml")
def sitemap():
    data = load_data()
    rendered_sitemap = flask.render_template("sitemap.xml", data=data)
    return flask.Response(rendered_sitemap, mimetype='application/xml')

@app.get("/license/<dataset>")
def license_dataset(dataset):
    data = load_data()
    if dataset not in data:
        return flask.abort(404)
    dataset_folder = os.path.join(DATASETS, dataset)
    license_file = os.path.join(dataset_folder, "license.txt")
    if not os.path.exists(license_file):
        return flask.Response("No license specified", mimetype='text/plain')
    with open(license_file, "r") as fp:
        license_template = fp.read()
    rendered_license = Template(license_template).render(**data[dataset])
    return flask.Response(rendered_license, mimetype='text/plain')


@app.route("/download/<dataset>", methods=["GET", "POST"])
def download_form(dataset):
    data = load_data()
    if dataset not in data:
        return flask.redirect('/')
    dataset_info = data[dataset]
    dataset_folder = os.path.join(DATASETS, dataset)
    zipfile = os.path.join(dataset_folder, "dataset.zip")
    has_url = bool(dataset_info.get("url", ""))
    has_zip = os.path.exists(zipfile)
    if not has_url and not has_zip:
        # Redirect to view page if neither url nor dataset.zip exists
        return flask.redirect(flask.url_for('view_dataset', dataset=dataset))
    dataset_folder = os.path.join(DATASETS, dataset)
    license_file = os.path.join(dataset_folder, "license.txt")
    if not os.path.exists(license_file):
        rendered_license = "No license specified"
    else:
        with open(license_file, "r") as fp:
            license_template = fp.read()
        rendered_license = Template(license_template).render(**data[dataset])

    if flask.request.method == "POST":
        title = flask.request.form.get('title', '').strip()
        name = flask.request.form.get('name', '').strip()
        if not name:
            return flask.render_template("download_form.html", rendered_license=rendered_license, **data[dataset])
        email = flask.request.form.get('email', '').strip()
        if not email:
            return flask.render_template("download_form.html", rendered_license=rendered_license, **data[dataset])
        affiliation = flask.request.form.get('affiliation', '').strip()
        license = flask.request.form.get('license', 'off')
        if license != 'on':
            return flask.render_template("download_form.html", rendered_license=rendered_license, **data[dataset])
        download_id = str(uuid.uuid4())

        with sqlite3.connect(DBFILE) as db:
            cursor = db.cursor()
            cursor.execute("INSERT INTO DOWNLOADS (id,dataset,title,name,email,affiliation) VALUES (?,?,?,?,?,?)",
                           (download_id, dataset, title, name, email, affiliation))
            db.commit()
        return flask.redirect(flask.url_for('download_dataset', dataset=dataset, id=download_id))
    else:
        return flask.render_template("download_form.html", rendered_license=rendered_license, **data[dataset])


def sizeof_fmt(num, suffix="B"):
    for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"):
        if abs(num) < 1024.0:
            if unit == "":
                return f"{int(num)}{unit}{suffix}"
            return f"{num:3.1f}{unit}{suffix}"
        num /= 1024.0
    return f"{num:.1f}Yi{suffix}"


@app.get("/download/<dataset>/<id>")
def download_dataset(dataset, id):
    data = load_data()
    if dataset not in data:
        return flask.redirect('/')
    dataset_folder = os.path.join(DATASETS, dataset)
    zipfile = os.path.join(dataset_folder, "dataset.zip")
    if not os.path.exists(zipfile):
        return flask.redirect('/')
    filesize = sizeof_fmt(os.stat(zipfile).st_size)
    download = {}
    with sqlite3.connect(DBFILE) as db:
        db.row_factory = sqlite3.Row
        cursor = db.cursor()
        cursor.execute("SELECT * FROM DOWNLOADS WHERE id=? AND dataset=?", (id, dataset))
        download = cursor.fetchall()
    if not download or len(download) != 1:
        return flask.redirect('/')
    download = download[0]
    expires = datetime.datetime.strptime(download['expires'], '%Y-%m-%d %H:%M:%S')
    if expires < datetime.datetime.now():
        return flask.redirect('/')
    url = flask.url_for('download_zip', dataset=dataset, id=id, _scheme="https", _external=True)
    return flask.render_template("download_dataset.html",
                                 url=url, expires=expires, download=download, filesize=filesize,
                                 **data[dataset])


@app.get("/download/<dataset>/<id>.zip")
def download_zip(dataset, id):
    data = load_data()
    if dataset not in data:
        return flask.redirect('/')
    dataset_folder = os.path.join(DATASETS, dataset)
    zipfile = os.path.join(dataset_folder, "dataset.zip")
    if not os.path.exists(zipfile):
        return flask.redirect('/')
    if os.path.exists(zipfile):
        filesize = sizeof_fmt(os.stat(zipfile).st_size)
    else:
        filesize = "N/A"
    download = {}
    with sqlite3.connect(DBFILE) as db:
        db.row_factory = sqlite3.Row
        cursor = db.cursor()
        cursor.execute("SELECT * FROM DOWNLOADS WHERE id=? AND dataset=?", (id, dataset))
        download = cursor.fetchall()
    if not download or len(download) != 1:
        return flask.redirect('/')
    download = download[0]
    expires = datetime.datetime.strptime(download['expires'], '%Y-%m-%d %H:%M:%S')
    if expires < datetime.datetime.now():
        return flask.redirect('/')
    return flask.send_from_directory(dataset_folder, "dataset.zip", 
                                     as_attachment=True, download_name=f"{dataset}.zip")


@app.get("/datasheet/<dataset>")
def datasheet_dataset(dataset):
    data = load_data()
    dataset_folder = os.path.join(DATASETS, dataset)
    datasheet_file = os.path.join(dataset_folder, "datasheet.pdf")
    if dataset not in data or not os.path.exists(datasheet_file):
        flask.abort(404)
    return flask.send_from_directory(dataset_folder, "datasheet.pdf", as_attachment=True, download_name=f"{dataset}_datasheet.pdf")


@app.get("/downloads")
def downloads():
    password = flask.request.args.get('password')
    if password != PASSWORD:
        return flask.redirect('/')

    downloads = []
    with sqlite3.connect(DBFILE) as db:
        db.row_factory = sqlite3.Row
        cursor = db.cursor()
        cursor.execute("SELECT * FROM DOWNLOADS")
        downloads = cursor.fetchall()

    return flask.render_template("downloads.html",
                                 title="Downloads", short_decription="List of all downloads", downloads=downloads)


@app.route("/image/<dataset>/<name>")
def serve_image(dataset, name):
    import mimetypes
    from flask import abort, send_file
    allowed_exts = {".png", ".jpg", ".jpeg", ".gif"}
    ext = os.path.splitext(name)[1].lower()
    if ext not in allowed_exts:
        abort(404)
    image_path = os.path.join(DATASETS, dataset, name)
    if not os.path.isfile(image_path):
        abort(404)
    mime = mimetypes.guess_type(image_path)[0] or "application/octet-stream"
    return send_file(image_path, mimetype=mime)


if __name__ == "__main__":
    app.run(debug=True, port=8080)