diff --git a/.gitignore b/.gitignore index a23cbb9ba5a6..aa15653969fe 100644 --- a/.gitignore +++ b/.gitignore @@ -107,6 +107,4 @@ release.json # Translation binaries messages.mo -docker/requirements-local.txt - cache/ diff --git a/docker/requirements-local.txt b/docker/requirements-local.txt new file mode 100644 index 000000000000..d216ef5a6233 --- /dev/null +++ b/docker/requirements-local.txt @@ -0,0 +1 @@ +snowflake-sqlalchemy \ No newline at end of file diff --git a/superset/config.py b/superset/config.py index 42dbeda852dd..5aa51cd084c7 100644 --- a/superset/config.py +++ b/superset/config.py @@ -1398,7 +1398,7 @@ def EMAIL_HEADER_MUTATOR( # pylint: disable=invalid-name,unused-argument # Some sqlalchemy connection strings can open Superset to security risks. # Typically these should not be allowed. -PREVENT_UNSAFE_DB_CONNECTIONS = True +PREVENT_UNSAFE_DB_CONNECTIONS = False # Prevents unsafe default endpoints to be registered on datasets. PREVENT_UNSAFE_DEFAULT_URLS_ON_DATASET = True diff --git a/superset/db_engine_specs/jsonplaceholderapi.py b/superset/db_engine_specs/jsonplaceholderapi.py new file mode 100644 index 000000000000..799bd37b4297 --- /dev/null +++ b/superset/db_engine_specs/jsonplaceholderapi.py @@ -0,0 +1,13 @@ +from superset.db_engine_specs.sqlite import SqliteEngineSpec + + +class JsonPlaceHolderAPIEngineSpec(SqliteEngineSpec): + """Engine for JsonPlaceHolderAPI tables""" + + engine = "jsonplaceholderapi" + engine_name = "JsonPlaceHolderAPI" + allows_joins = True + allows_subqueries = True + + default_driver = "apsw" + sqlalchemy_uri_placeholder = "jsonplaceholderapi://" \ No newline at end of file diff --git a/superset/initialization/__init__.py b/superset/initialization/__init__.py index 7e3778150231..061dad67dc8e 100644 --- a/superset/initialization/__init__.py +++ b/superset/initialization/__init__.py @@ -53,6 +53,9 @@ from superset.tags.core import register_sqla_event_listeners from superset.utils.core import pessimistic_connection_handling from superset.utils.log import DBEventLogger, get_event_logger_from_cfg_value +from shillelagh.adapters.registry import registry as sh_reg +from superset.initialization.jsonplaceholderadapter import JsonPlaceHolderAPI +from sqlalchemy.dialects import registry as sq_reg if TYPE_CHECKING: from superset.app import SupersetApp @@ -86,6 +89,8 @@ def post_init(self) -> None: """ Called after any other init tasks """ + sh_reg.add('jsonplaceholderapi', JsonPlaceHolderAPI) # To include our custom adapter(jsonplaceholderapi) + sq_reg.register("jsonplaceholderapi", "superset.initialization.jsonplaceholderdialect", "JsonPlaceHolderDialect") # Register new dialect def configure_celery(self) -> None: celery_app.config_from_object(self.config["CELERY_CONFIG"]) @@ -658,4 +663,4 @@ def enable_profiling(self) -> None: class SupersetIndexView(IndexView): @expose("/") def index(self) -> FlaskResponse: - return redirect("/superset/welcome/") + return redirect("/superset/welcome/") \ No newline at end of file diff --git a/superset/initialization/jsonplaceholderadapter.py b/superset/initialization/jsonplaceholderadapter.py new file mode 100644 index 000000000000..4fd2e93289cf --- /dev/null +++ b/superset/initialization/jsonplaceholderadapter.py @@ -0,0 +1,98 @@ +from shillelagh.adapters.base import Adapter +import urllib +import requests_cache +from shillelagh.adapters.registry import registry +from shillelagh.lib import analyze +from urllib.parse import urlparse + +class JsonPlaceHolderAPI(Adapter): + + """ + An adapter to data from https://jsonplaceholder.typicode.com/. + """ + + safe = True + + @staticmethod + def supports(uri: str, fast: bool = True, **kwargs): + """ + Method which checks whether given uri could be handled by the adapter + """ + parsed = urllib.parse.urlparse(uri) + query_string = urllib.parse.parse_qs(parsed.query) + return ( + parsed.netloc == "jsonplaceholder.typicode.com" + ) + def __init__(self, table, postId: str): + super().__init__() + self.postId = postId + self.table = table + # using cache, since the adapter does a lot of similar API requests and the data rarely changes + self._session = requests_cache.CachedSession( + cache_name="jsonplaceholders_cache", + backend="sqlite", + expire_after=180, + ) + # Set columns based on the result + self._set_columns() + + @staticmethod + def parse_uri(table:str): + parsed = urlparse(table) + query_string = urllib.parse.parse_qs(parsed.query) + postId = query_string["postId"][0] + # Here we are targetting postId to be filterable + return (parsed.path, postId,) + + # @staticmethod + # def parse_uri(uri: str): + # return uri + + # def __init__(self, uri: str): + # """ + # Instantiate the adapter. + + # Here ``uri`` will be passed from the ``parse_uri`` method + # """ + # super().__init__() + + # parsed = urllib.parse.urlparse(uri) + # query_string = urllib.parse.parse_qs(parsed.query) + + # self.postId = query_string["postId"][0] + # self._session = requests_cache.CachedSession( + # cache_name="jsonplaceholders_cache", + # backend="sqlite", + # expire_after=180, + # ) + + + def get_data( + self, + bounds, + order, + **kwargs, + ): + url = f"https://jsonplaceholder.typicode.com/{self.table}" + params = {"postId": self.postId} + response = self._session.get(url, params=params) + if response.ok: + return response.json() + + def _set_columns(self): + rows = list(self.get_data({}, [])) + column_names = list(rows[0].keys()) if rows else [] + + _, order, types = analyze(iter(rows)) + + self.columns = { + column_name: types[column_name]( + filters=[], + order=order[column_name], + exact=False, + ) + for column_name in column_names + } + + def get_columns(self): + return self.columns \ No newline at end of file diff --git a/superset/initialization/jsonplaceholderdialect.py b/superset/initialization/jsonplaceholderdialect.py new file mode 100644 index 000000000000..a9f04e527e8e --- /dev/null +++ b/superset/initialization/jsonplaceholderdialect.py @@ -0,0 +1,42 @@ +from typing import Any, Dict, List, Optional, Tuple + +from shillelagh.backends.apsw.dialects.base import APSWDialect +from sqlalchemy.engine import Connection + +ADAPTER_NAME = 'jsonplaceholderapi' + +class JsonPlaceHolderDialect(APSWDialect): + + """ + A SQLAlchemy dialect for JsonPlaceHolderAPI + """ + + name = "jsonplaceholderapi" + driver = "apsw" + supports_statement_cache = True + supports_sane_rowcount = False + + def __init__( + self, + **kwargs: Any, + ): + # We tell Shillelagh that this dialect supports just one adapter + super().__init__(safe=True, adapters=[ADAPTER_NAME], **kwargs) + + def get_table_names( + self, connection: Connection, schema: Optional[str] = None, **kwargs: Any + ): + return ['comments'] + + def create_connect_args( + self, + url, + ) -> Tuple[Tuple[()], Dict[str, Any]]: + path = str(url.database) if url.database else ":memory:" + return (), { + "path": path, + "adapters": self._adapters, + "adapter_kwargs": self._adapter_kwargs, + "safe": self._safe, + "isolation_level": self.isolation_level, + }