diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index 6512af33..f2c9c406 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -33,6 +33,9 @@ class Settings(BaseSettings): vpn_credential_refresh_hours: int = 24 + # Bucket used to store configuration files + config_bucket: str = "" + # Where the geoip DBs are downloaded to geoip_db_dir: str = "/var/lib/ooni/geoip" # Ooniprobe only diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py index 4a6dc663..83504d74 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/utils.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/utils.py @@ -9,9 +9,11 @@ from datetime import datetime, timezone import itertools import logging -from typing import Dict, List, Mapping, TypedDict, Tuple +from typing import List, TypedDict, Tuple +import io from fastapi import Request +from mypy_boto3_s3 import S3Client from sqlalchemy.orm import Session import pem import httpx @@ -145,7 +147,7 @@ def lookup_probe_network(ipaddr: str, asn_reader: ASNReaderDep) -> Tuple[str, st "AS{}".format(resp.autonomous_system_number), resp.autonomous_system_organization or "0", ) - + def get_first_ip(headers: str) -> str: """ parse the first ip from a comma-separated list of ips encoded as a string @@ -154,5 +156,14 @@ def get_first_ip(headers: str) -> str: in: '123.123.123, 1.1.1.1' out: '123.123.123' """ + return headers.partition(',')[0] + +def read_file(s3_client : S3Client, bucket: str, file : str) -> str: + """ + Reads the content of `file` within `bucket` into a string - return headers.partition(',')[0] \ No newline at end of file + Useful for reading config files from the s3 bucket + """ + buff = io.BytesIO() + s3_client.download_fileobj(bucket, file, buff) + return buff.getvalue().decode() \ No newline at end of file