From 592d91610467f4e5f8c6a728443e08991d22d8c5 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 22 Feb 2024 19:46:56 +0100 Subject: [PATCH 1/4] Reshaping: Converge content into the layout of crate-clients-tools - Integrations - Load and Export - System Metrics - Data Visualization - Business Intelligence - Machine Learning - Software Testing - Application Domains - Time Series Data - Machine Learning While the content is in a single place only, in order to avoid redundancy, the separate "Application Domains" section can be used to provide an alternative categorization/grouping perspective. In this section, you can cherry-pick specific content sections, in order to provide better guidance into corresponding topics, optionally selecting from content within "Integrations", or adding dedicated content like "Time Series Data", which just does not fit into "Integrations" well. --- docs/domain/index.md | 15 ++ .../timeseries/generate/cli.rst | 0 .../timeseries/generate/go.rst | 0 .../timeseries/generate/index.rst | 0 .../timeseries/generate/node.rst | 0 .../timeseries/generate/python.rst | 0 docs/{topic => domain}/timeseries/index.rst | 0 .../timeseries/normalize-intervals.rst | 0 docs/index.html | 13 -- docs/index.md | 100 +++++++++-- docs/integrate/bi/index.md | 17 ++ .../bi}/powerbi-desktop.rst | 0 .../bi}/powerbi-gateway.rst | 0 docs/integrate/{ => etl}/azure-functions.rst | 0 docs/integrate/etl/index.md | 149 +++++++++++++++ docs/integrate/{ => etl}/kafka-connect.rst | 0 docs/integrate/{ => etl}/mongodb.rst | 0 docs/integrate/{ => etl}/mysql.rst | 0 docs/integrate/{ => etl}/streamsets.rst | 0 docs/integrate/index.md | 170 ++---------------- docs/integrate/metrics/index.md | 28 +++ docs/{topic => integrate}/ml/index.md | 3 +- docs/{topic => integrate}/ml/tensorflow.rst | 0 docs/{topic => integrate}/testing.md | 0 .../visualize}/grafana.rst | 0 .../analysis => integrate/visualize}/index.md | 16 +- .../visualize}/metabase.rst | 0 .../analysis => integrate/visualize}/r.rst | 0 28 files changed, 315 insertions(+), 196 deletions(-) create mode 100644 docs/domain/index.md rename docs/{topic => domain}/timeseries/generate/cli.rst (100%) rename docs/{topic => domain}/timeseries/generate/go.rst (100%) rename docs/{topic => domain}/timeseries/generate/index.rst (100%) rename docs/{topic => domain}/timeseries/generate/node.rst (100%) rename docs/{topic => domain}/timeseries/generate/python.rst (100%) rename docs/{topic => domain}/timeseries/index.rst (100%) rename docs/{topic => domain}/timeseries/normalize-intervals.rst (100%) delete mode 100644 docs/index.html create mode 100644 docs/integrate/bi/index.md rename docs/{topic/analysis => integrate/bi}/powerbi-desktop.rst (100%) rename docs/{topic/analysis => integrate/bi}/powerbi-gateway.rst (100%) rename docs/integrate/{ => etl}/azure-functions.rst (100%) create mode 100644 docs/integrate/etl/index.md rename docs/integrate/{ => etl}/kafka-connect.rst (100%) rename docs/integrate/{ => etl}/mongodb.rst (100%) rename docs/integrate/{ => etl}/mysql.rst (100%) rename docs/integrate/{ => etl}/streamsets.rst (100%) create mode 100644 docs/integrate/metrics/index.md rename docs/{topic => integrate}/ml/index.md (98%) rename docs/{topic => integrate}/ml/tensorflow.rst (100%) rename docs/{topic => integrate}/testing.md (100%) rename docs/{topic/analysis => integrate/visualize}/grafana.rst (100%) rename docs/{topic/analysis => integrate/visualize}/index.md (94%) rename docs/{topic/analysis => integrate/visualize}/metabase.rst (100%) rename docs/{topic/analysis => integrate/visualize}/r.rst (100%) diff --git a/docs/domain/index.md b/docs/domain/index.md new file mode 100644 index 00000000..85a5d863 --- /dev/null +++ b/docs/domain/index.md @@ -0,0 +1,15 @@ +(domain)= +(domains)= + +# Application Domains + +This section of the documentation includes tutorials and guidelines about +how to use CrateDB optimally, related to different topic domains. + + +```{toctree} +:maxdepth: 3 + +timeseries/index +../integrate/ml/index +``` diff --git a/docs/topic/timeseries/generate/cli.rst b/docs/domain/timeseries/generate/cli.rst similarity index 100% rename from docs/topic/timeseries/generate/cli.rst rename to docs/domain/timeseries/generate/cli.rst diff --git a/docs/topic/timeseries/generate/go.rst b/docs/domain/timeseries/generate/go.rst similarity index 100% rename from docs/topic/timeseries/generate/go.rst rename to docs/domain/timeseries/generate/go.rst diff --git a/docs/topic/timeseries/generate/index.rst b/docs/domain/timeseries/generate/index.rst similarity index 100% rename from docs/topic/timeseries/generate/index.rst rename to docs/domain/timeseries/generate/index.rst diff --git a/docs/topic/timeseries/generate/node.rst b/docs/domain/timeseries/generate/node.rst similarity index 100% rename from docs/topic/timeseries/generate/node.rst rename to docs/domain/timeseries/generate/node.rst diff --git a/docs/topic/timeseries/generate/python.rst b/docs/domain/timeseries/generate/python.rst similarity index 100% rename from docs/topic/timeseries/generate/python.rst rename to docs/domain/timeseries/generate/python.rst diff --git a/docs/topic/timeseries/index.rst b/docs/domain/timeseries/index.rst similarity index 100% rename from docs/topic/timeseries/index.rst rename to docs/domain/timeseries/index.rst diff --git a/docs/topic/timeseries/normalize-intervals.rst b/docs/domain/timeseries/normalize-intervals.rst similarity index 100% rename from docs/topic/timeseries/normalize-intervals.rst rename to docs/domain/timeseries/normalize-intervals.rst diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 4d9ac6c2..00000000 --- a/docs/index.html +++ /dev/null @@ -1,13 +0,0 @@ -{ - "ok" : true, - "status" : 200, - "name" : "Großer Bettelwurf", - "cluster_name" : "crate", - "version" : { - "number" : "4.1.5", - "build_hash" : "135aede817dc392095e10e7a05fb7248d0140c14", - "build_timestamp" : "2020-04-24T09:00:14Z", - "build_snapshot" : false, - "lucene_version" : "8.4.0" - } -} diff --git a/docs/index.md b/docs/index.md index 90858b2b..b7f367ad 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,19 +90,61 @@ use-cases. -## Application Domains +## Integrations -Native features of CrateDB paired with auxiliary software components provide -optimal coverage of different application domains. +Learn how to use CrateDB with 3rd-party software applications, libraries, and +frameworks. ::::{grid} 3 :padding: 0 -:::{grid-item-card} Analysis and Visualization +:::{grid-item-card} ETL +:link: etl +:link-type: ref +:link-alt: Load and export data into/from CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`transform;1.3em` +::: + + +:::{grid-item-card} Metrics +:link: metrics +:link-type: ref +:link-alt: CrateDB with metrics collection agents, brokers, and stores +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`query_stats;1.3em` +::: + + +:::{grid-item-card} Data Visualization :link: analysis :link-type: ref -:link-alt: Data Analysis and Visualization with CrateDB +:link-alt: Data visualization with CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`bar_chart;1.3em` +::: + + +:::{grid-item-card} Business Intelligence +:link: bi +:link-type: ref +:link-alt: Analyse information with CrateDB :padding: 3 :text-align: center :class-card: sd-pt-3 @@ -113,6 +155,45 @@ optimal coverage of different application domains. ::: +:::{grid-item-card} Machine Learning +:link: machine-learning +:link-type: ref +:link-alt: Machine Learning with CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`model_training;1.3em` +::: + + +:::{grid-item-card} Software Testing +:link: testing +:link-type: ref +:link-alt: Software testing with CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`integration_instructions;1.3em` +::: + + +:::: + + +## Application Domains + +Learn how to apply CrateDB's features to optimally cover different +application and topic domains. + +::::{grid} 3 +:padding: 0 + :::{grid-item-card} Time Series Data :link: timeseries :link-type: ref @@ -140,11 +221,9 @@ optimal coverage of different application domains. {material-outlined}`model_training;1.3em` ::: - :::: - ```{toctree} :hidden: @@ -152,12 +231,9 @@ install/index getting-started/index admin/index integrate/index - reference-architectures/index -topic/analysis/index -topic/timeseries/index -topic/ml/index -topic/testing + +domain/index ``` diff --git a/docs/integrate/bi/index.md b/docs/integrate/bi/index.md new file mode 100644 index 00000000..3f040e14 --- /dev/null +++ b/docs/integrate/bi/index.md @@ -0,0 +1,17 @@ +(analysis)= +(bi)= + +# Business Intelligence + +Integrations of CrateDB with other tools, specifically related to business +analytics and intelligence software. + + +## PowerBI + +```{toctree} +:maxdepth: 1 + +powerbi-desktop +powerbi-gateway +``` diff --git a/docs/topic/analysis/powerbi-desktop.rst b/docs/integrate/bi/powerbi-desktop.rst similarity index 100% rename from docs/topic/analysis/powerbi-desktop.rst rename to docs/integrate/bi/powerbi-desktop.rst diff --git a/docs/topic/analysis/powerbi-gateway.rst b/docs/integrate/bi/powerbi-gateway.rst similarity index 100% rename from docs/topic/analysis/powerbi-gateway.rst rename to docs/integrate/bi/powerbi-gateway.rst diff --git a/docs/integrate/azure-functions.rst b/docs/integrate/etl/azure-functions.rst similarity index 100% rename from docs/integrate/azure-functions.rst rename to docs/integrate/etl/azure-functions.rst diff --git a/docs/integrate/etl/index.md b/docs/integrate/etl/index.md new file mode 100644 index 00000000..75a45ba2 --- /dev/null +++ b/docs/integrate/etl/index.md @@ -0,0 +1,149 @@ +(etl)= + +# Load and Export + +You have a variety of options to connect and integrate with 3rd-party +ETL applications, mostly using [CrateDB's PostgreSQL interface]. + +This documentation section lists corresponding ETL applications and +frameworks which can be used together with CrateDB, and outlines how +to use them optimally. + + +## Apache Airflow / Astronomer + +A set of starter tutorials. + +- [Automating the import of Parquet files with Apache Airflow] +- [Updating stock market data automatically with CrateDB and Apache Airflow] +- [Automating stock data collection and storage with CrateDB and Apache Airflow] + +A set of elaborated tutorials, including blueprint implementations. + +- [Automating export of CrateDB data to S3 using Apache Airflow] +- [Implementing a data retention policy in CrateDB using Apache Airflow] +- [CrateDB and Apache Airflow: Building a data ingestion pipeline] +- [Building a hot and cold storage data retention policy in CrateDB with Apache Airflow] + +Tutorials and resources about configuring the managed variants, Astro and CrateDB Cloud. + +- [ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud] +- [ETL pipeline using Apache Airflow with CrateDB (Source)] +- [Run an ETL pipeline with CrateDB and data quality checks] + + +## Apache Flink + +- {ref}`kafka-connect` +- [Build a data ingestion pipeline using Kafka, Flink, and CrateDB] +- [Community Day: Stream processing with Apache Flink and CrateDB] +- [Executable stack: Apache Kafka, Apache Flink, and CrateDB] + + +## Apache Kafka + +- [Data Ingestion using Kafka and Kafka Connect] +- [Executable stack: Apache Kafka, Apache Flink, and CrateDB] +- [Tutorial: Replicating data to CrateDB with Debezium and Kafka] + + +## Azure Functions + +- {ref}`azure-functions` + + +## dbt + +- [Using dbt with CrateDB] + + +## Debezium + +- [Tutorial: Replicating data to CrateDB with Debezium and Kafka] +- [Webinar: How to replicate data from other databases to CrateDB with Debezium and Kafka] + + +## Kestra + +- [Setting up data pipelines with CrateDB and Kestra] + + +## MongoDB + +- {ref}`integrate-mongodb` + + +## MySQL + +- {ref}`integrate-mysql` + + +## Node-RED + +- [Ingesting MQTT messages into CrateDB using Node-RED] +- [Automating recurrent CrateDB queries using Node-RED] + + +## Singer / Meltano + +- [meltano-target-cratedb] +- [meltano-tap-cratedb] +- [Examples about working with CrateDB and Meltano] + +🚧 _Please note these adapters are a work in progress._ 🚧 + + +## SQL Server Integration Services + +A demo project which uses SSIS and ODBC to read and write data from CrateDB: + +- [Using SQL Server Integration Services with CrateDB] + + +## StreamSets + +- {ref}`streamsets` + + +[Automating recurrent CrateDB queries using Node-RED]: https://community.cratedb.com/t/automating-recurrent-cratedb-queries/788 +[Automating export of CrateDB data to S3 using Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 +[Automating stock data collection and storage with CrateDB and Apache Airflow]: https://community.cratedb.com/t/automating-stock-data-collection-and-storage-with-cratedb-and-apache-airflow/990 +[Automating the import of Parquet files with Apache Airflow]: https://community.cratedb.com/t/automating-the-import-of-parquet-files-with-apache-airflow/1247 +[Build a data ingestion pipeline using Kafka, Flink, and CrateDB]: https://dev.to/crate/build-a-data-ingestion-pipeline-using-kafka-flink-and-cratedb-1h5o +[Building a hot and cold storage data retention policy in CrateDB with Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-hot-cold-storage-data-retention-policy/934 +[Community Day: Stream processing with Apache Flink and CrateDB]: https://cratedb.com/blog/cratedb-community-day-2nd-edition-summary-and-highlights +[CrateDB and Apache Airflow: Building a data ingestion pipeline]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-data-ingestion-pipeline/926 +[CrateDB's PostgreSQL interface]: inv:crate-reference#interface-postgresql +[Data Ingestion using Kafka and Kafka Connect]: https://cratedb.com/docs/crate/howtos/en/latest/integrations/kafka-connect.html +[ETL pipeline using Apache Airflow with CrateDB (Source)]: https://github.com/astronomer/astro-cratedb-blogpost +[ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud]: https://www.astronomer.io/blog/run-etlelt-with-airflow-and-cratedb/ +[Examples about working with CrateDB and Meltano]: https://github.com/crate/cratedb-examples/tree/amo/meltano/framework/singer-meltano +[Executable stack: Apache Kafka, Apache Flink, and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/apache-kafka-flink +[Implementing a data retention policy in CrateDB using Apache Airflow]: https://community.cratedb.com/t/implementing-a-data-retention-policy-in-cratedb-using-apache-airflow/913 +[Ingesting MQTT messages into CrateDB using Node-RED]: https://community.cratedb.com/t/ingesting-mqtt-messages-into-cratedb-using-node-red/803 +[meltano-tap-cratedb]: https://github.com/crate-workbench/meltano-tap-cratedb +[meltano-target-cratedb]: https://github.com/crate-workbench/meltano-target-cratedb +[Run an ETL pipeline with CrateDB and data quality checks]: https://registry.astronomer.io/dags/etl_pipeline/ +[Setting up data pipelines with CrateDB and Kestra]: https://community.cratedb.com/t/setting-up-data-pipelines-with-cratedb-and-kestra-io/1400 +[Tutorial: Replicating data to CrateDB with Debezium and Kafka]: https://community.cratedb.com/t/replicating-data-to-cratedb-with-debezium-and-kafka/1388 +[Updating stock market data automatically with CrateDB and Apache Airflow]: https://community.cratedb.com/t/updating-stock-market-data-automatically-with-cratedb-and-apache-airflow/1304 +[Using dbt with CrateDB]: https://community.cratedb.com/t/using-dbt-with-cratedb/1566 +[Using SQL Server Integration Services with CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/microsoft-ssis +[Webinar: How to replicate data from other databases to CrateDB with Debezium and Kafka]: https://cratedb.com/resources/webinars/lp-wb-debezium-kafka + + + +```{toctree} +:hidden: + +mongodb +mysql +``` + +```{toctree} +:hidden: + +kafka-connect +azure-functions +streamsets +``` diff --git a/docs/integrate/kafka-connect.rst b/docs/integrate/etl/kafka-connect.rst similarity index 100% rename from docs/integrate/kafka-connect.rst rename to docs/integrate/etl/kafka-connect.rst diff --git a/docs/integrate/mongodb.rst b/docs/integrate/etl/mongodb.rst similarity index 100% rename from docs/integrate/mongodb.rst rename to docs/integrate/etl/mongodb.rst diff --git a/docs/integrate/mysql.rst b/docs/integrate/etl/mysql.rst similarity index 100% rename from docs/integrate/mysql.rst rename to docs/integrate/etl/mysql.rst diff --git a/docs/integrate/streamsets.rst b/docs/integrate/etl/streamsets.rst similarity index 100% rename from docs/integrate/streamsets.rst rename to docs/integrate/etl/streamsets.rst diff --git a/docs/integrate/index.md b/docs/integrate/index.md index c85c646e..7a27ad94 100644 --- a/docs/integrate/index.md +++ b/docs/integrate/index.md @@ -5,150 +5,21 @@ You have a variety of options to connect and integrate 3rd-party applications, mostly using [CrateDB's PostgreSQL interface]. -This documentation section lists frameworks and applications which can -be used together with CrateDB, and outlines how to use them optimally. +This documentation section lists applications, frameworks, and libraries, +which can be used together with CrateDB, and outlines how to use them +optimally. -## Apache Airflow / Astronomer - -A set of starter tutorials. - -- [Automating the import of Parquet files with Apache Airflow] -- [Updating stock market data automatically with CrateDB and Apache Airflow] -- [Automating stock data collection and storage with CrateDB and Apache Airflow] - -A set of elaborated tutorials, including blueprint implementations. - -- [Automating export of CrateDB data to S3 using Apache Airflow] -- [Implementing a data retention policy in CrateDB using Apache Airflow] -- [CrateDB and Apache Airflow: Building a data ingestion pipeline] -- [Building a hot and cold storage data retention policy in CrateDB with Apache Airflow] - -Tutorials and resources about configuring the managed variants, Astro and CrateDB Cloud. - -- [ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud] -- [ETL pipeline using Apache Airflow with CrateDB (Source)] -- [Run an ETL pipeline with CrateDB and data quality checks] - - -## Apache Flink - -- {ref}`kafka-connect` -- [Build a data ingestion pipeline using Kafka, Flink, and CrateDB] -- [Community Day: Stream processing with Apache Flink and CrateDB] -- [Executable stack: Apache Kafka, Apache Flink, and CrateDB] - - -## Apache Kafka - -- [Data Ingestion using Kafka and Kafka Connect] -- [Executable stack: Apache Kafka, Apache Flink, and CrateDB] -- [Tutorial: Replicating data to CrateDB with Debezium and Kafka] - - -## Azure Functions - -- {ref}`azure-functions` - - -## dbt - -- [Using dbt with CrateDB] - - -## Debezium - -- [Tutorial: Replicating data to CrateDB with Debezium and Kafka] -- [Webinar: How to replicate data from other databases to CrateDB with Debezium and Kafka] - - -## Kestra - -- [Setting up data pipelines with CrateDB and Kestra] - - -## MongoDB - -- {ref}`integrate-mongodb` - - -## MySQL - -- {ref}`integrate-mysql` - - -## Node-RED - -- [Ingesting MQTT messages into CrateDB using Node-RED] -- [Automating recurrent CrateDB queries using Node-RED] - - -## Prometheus - -- [CrateDB Prometheus Adapter] -- [Getting Started With Prometheus and CrateDB for Long-Term Storage] -- [Storing long-term metrics with Prometheus in CrateDB] -- [Webinar: Using Prometheus and Grafana with CrateDB Cloud] - - -## Singer / Meltano - -- [meltano-target-cratedb] -- [meltano-tap-cratedb] -- [Examples about working with CrateDB and Meltano] - -🚧 _Please note these adapters are a work in progress._ 🚧 - - -## SQL Server Integration Services - -A demo project which uses SSIS and ODBC to read and write data from CrateDB: - -- [Using SQL Server Integration Services with CrateDB] - - -## Streamsets - -- {ref}`streamsets` - - -## Telegraf - -- [Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics] - - -[Automating recurrent CrateDB queries using Node-RED]: https://community.cratedb.com/t/automating-recurrent-cratedb-queries/788 -[Automating export of CrateDB data to S3 using Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 -[Automating stock data collection and storage with CrateDB and Apache Airflow]: https://community.cratedb.com/t/automating-stock-data-collection-and-storage-with-cratedb-and-apache-airflow/990 -[Automating the import of Parquet files with Apache Airflow]: https://community.cratedb.com/t/automating-the-import-of-parquet-files-with-apache-airflow/1247 -[Build a data ingestion pipeline using Kafka, Flink, and CrateDB]: https://dev.to/crate/build-a-data-ingestion-pipeline-using-kafka-flink-and-cratedb-1h5o -[Building a hot and cold storage data retention policy in CrateDB with Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-hot-cold-storage-data-retention-policy/934 -[Community Day: Stream processing with Apache Flink and CrateDB]: https://cratedb.com/blog/cratedb-community-day-2nd-edition-summary-and-highlights -[CrateDB and Apache Airflow: Building a data ingestion pipeline]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-data-ingestion-pipeline/926 -[CrateDB's PostgreSQL interface]: inv:crate-reference:*:label#interface-postgresql -[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter -[Data Ingestion using Kafka and Kafka Connect]: https://cratedb.com/docs/crate/howtos/en/latest/integrations/kafka-connect.html -[ETL pipeline using Apache Airflow with CrateDB (Source)]: https://github.com/astronomer/astro-cratedb-blogpost -[ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud]: https://www.astronomer.io/blog/run-etlelt-with-airflow-and-cratedb/ -[Examples about working with CrateDB and Meltano]: https://github.com/crate/cratedb-examples/tree/amo/meltano/framework/singer-meltano -[Executable stack: Apache Kafka, Apache Flink, and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/apache-kafka-flink -[Getting Started With Prometheus and CrateDB for Long-Term Storage]: https://cratedb.com/blog/getting-started-prometheus-cratedb-long-term-storage -[Implementing a data retention policy in CrateDB using Apache Airflow]: https://community.cratedb.com/t/implementing-a-data-retention-policy-in-cratedb-using-apache-airflow/913 -[Ingesting MQTT messages into CrateDB using Node-RED]: https://community.cratedb.com/t/ingesting-mqtt-messages-into-cratedb-using-node-red/803 -[meltano-tap-cratedb]: https://github.com/crate-workbench/meltano-tap-cratedb -[meltano-target-cratedb]: https://github.com/crate-workbench/meltano-target-cratedb -[Overview of CrateDB integration tutorials]: https://community.cratedb.com/t/overview-of-cratedb-integration-tutorials/1015 -[Run an ETL pipeline with CrateDB and data quality checks]: https://registry.astronomer.io/dags/etl_pipeline/ -[Setting up data pipelines with CrateDB and Kestra]: https://community.cratedb.com/t/setting-up-data-pipelines-with-cratedb-and-kestra-io/1400 -[Storing long-term metrics with Prometheus in CrateDB]: https://community.cratedb.com/t/storing-long-term-metrics-with-prometheus-in-cratedb/1012 -[Tutorial: Replicating data to CrateDB with Debezium and Kafka]: https://community.cratedb.com/t/replicating-data-to-cratedb-with-debezium-and-kafka/1388 -[Updating stock market data automatically with CrateDB and Apache Airflow]: https://community.cratedb.com/t/updating-stock-market-data-automatically-with-cratedb-and-apache-airflow/1304 -[Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics]: https://cratedb.com/blog/use-cratedb-with-telegraf-an-agent-for-collecting-reporting-metrics -[Using dbt with CrateDB]: https://community.cratedb.com/t/using-dbt-with-cratedb/1566 -[Using SQL Server Integration Services with CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/microsoft-ssis -[Webinar: How to replicate data from other databases to CrateDB with Debezium and Kafka]: https://cratedb.com/resources/webinars/lp-wb-debezium-kafka -[Webinar: Using Prometheus and Grafana with CrateDB Cloud]: https://cratedb.com/resources/webinars/lp-wb-prometheus-grafana - +```{toctree} +:maxdepth: 2 + +etl/index +metrics/index +visualize/index +bi/index +ml/index +testing +``` :::{tip} Please also visit the [Overview of CrateDB integration tutorials]. @@ -156,17 +27,6 @@ Please also visit the [Overview of CrateDB integration tutorials]. -```{toctree} -:hidden: - -mongodb -mysql -``` -```{toctree} -:hidden: - -kafka-connect -azure-functions -streamsets -``` +[CrateDB's PostgreSQL interface]: inv:crate-reference#interface-postgresql +[Overview of CrateDB integration tutorials]: https://community.cratedb.com/t/overview-of-cratedb-integration-tutorials/1015 diff --git a/docs/integrate/metrics/index.md b/docs/integrate/metrics/index.md new file mode 100644 index 00000000..123bc339 --- /dev/null +++ b/docs/integrate/metrics/index.md @@ -0,0 +1,28 @@ +(metrics)= + +# System Metrics + +CrateDB integrations with metrics collection agents, brokers, and stores. + +This documentation section lists applications and daemons which can +be used together with CrateDB, and outlines how to use them optimally. + + +## Prometheus + +- [CrateDB Prometheus Adapter] +- [Getting Started With Prometheus and CrateDB for Long-Term Storage] +- [Storing long-term metrics with Prometheus in CrateDB] +- [Webinar: Using Prometheus and Grafana with CrateDB Cloud] + + +## Telegraf + +- [Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics] + + +[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter +[Getting Started With Prometheus and CrateDB for Long-Term Storage]: https://cratedb.com/blog/getting-started-prometheus-cratedb-long-term-storage +[Storing long-term metrics with Prometheus in CrateDB]: https://community.cratedb.com/t/storing-long-term-metrics-with-prometheus-in-cratedb/1012 +[Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics]: https://cratedb.com/blog/use-cratedb-with-telegraf-an-agent-for-collecting-reporting-metrics +[Webinar: Using Prometheus and Grafana with CrateDB Cloud]: https://cratedb.com/resources/webinars/lp-wb-prometheus-grafana diff --git a/docs/topic/ml/index.md b/docs/integrate/ml/index.md similarity index 98% rename from docs/topic/ml/index.md rename to docs/integrate/ml/index.md index 91c54f5f..5feb62a1 100644 --- a/docs/topic/ml/index.md +++ b/docs/integrate/ml/index.md @@ -3,8 +3,7 @@ # Machine Learning -This section of the documentation covers integrations of CrateDB with machine -learning frameworks and tools. +Guidelines about integrating CrateDB with machine learning frameworks and tools. (langchain)= ## LangChain diff --git a/docs/topic/ml/tensorflow.rst b/docs/integrate/ml/tensorflow.rst similarity index 100% rename from docs/topic/ml/tensorflow.rst rename to docs/integrate/ml/tensorflow.rst diff --git a/docs/topic/testing.md b/docs/integrate/testing.md similarity index 100% rename from docs/topic/testing.md rename to docs/integrate/testing.md diff --git a/docs/topic/analysis/grafana.rst b/docs/integrate/visualize/grafana.rst similarity index 100% rename from docs/topic/analysis/grafana.rst rename to docs/integrate/visualize/grafana.rst diff --git a/docs/topic/analysis/index.md b/docs/integrate/visualize/index.md similarity index 94% rename from docs/topic/analysis/index.md rename to docs/integrate/visualize/index.md index 1fc731dc..ef21fe8f 100644 --- a/docs/topic/analysis/index.md +++ b/docs/integrate/visualize/index.md @@ -1,10 +1,8 @@ -(analysis)= (visualization)= -# Data Analysis and Visualization +# Data Visualization -This section of the documentation covers integrations of CrateDB with other -tools, specifically related to data analysis and visualization. +Guidelines about data analysis and visualization with CrateDB. ## Apache Superset / Preset @@ -73,16 +71,6 @@ into a CrateDB Cloud database cluster. - Alternatively, you are welcome to explore the canonical [Dash Examples]. -## PowerBI - -```{toctree} -:maxdepth: 1 - -powerbi-desktop -powerbi-gateway -``` - - ## R ```{toctree} diff --git a/docs/topic/analysis/metabase.rst b/docs/integrate/visualize/metabase.rst similarity index 100% rename from docs/topic/analysis/metabase.rst rename to docs/integrate/visualize/metabase.rst diff --git a/docs/topic/analysis/r.rst b/docs/integrate/visualize/r.rst similarity index 100% rename from docs/topic/analysis/r.rst rename to docs/integrate/visualize/r.rst From 5da5f2f1f074eacc78efafb6360399618d3c49a7 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 22 Feb 2024 20:16:42 +0100 Subject: [PATCH 2/4] Chore: Improve wording on main index / home page --- docs/index.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index b7f367ad..61171d52 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,9 +3,9 @@ (howtos)= (tutorials)= -# CrateDB Guides and Tutorials +# The CrateDB Guide -About how to use CrateDB and CrateDB Cloud in practice. +Guides and tutorials about how to use CrateDB and CrateDB Cloud in practice. CrateDB is a distributed and scalable SQL database for storing and analyzing massive amounts of data in near real-time, even with complex queries. It is @@ -242,8 +242,13 @@ Please also visit the [Overview of CrateDB integration tutorials]. ::: :::{seealso} -CrateDB and its documentation are open source projects. We host the source code and -issue tracker on [GitHub]. +CrateDB and its documentation are open source projects. +Contributions to the pages in this section and subsections are much appreciated. +If you can spot a flaw, or would like to contribute additional content, you +are most welcome. + +You will find corresponding links within the topmost right navigation element +on each page, linking to the relevant page where this project is hosted on [GitHub]. ::: From 500c42b6854c2cffbdbbe6129a172344b6d823df Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 22 Feb 2024 20:17:39 +0100 Subject: [PATCH 3/4] Reshaping: Properly order main section pillars - Reference Architectures - Application Domains - Integrations --- docs/index.md | 75 ++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/docs/index.md b/docs/index.md index 61171d52..e54bfccf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -89,6 +89,43 @@ use-cases. +## Application Domains + +Learn how to apply CrateDB's features to optimally cover use-cases in different +application and topic domains. + +::::{grid} 3 +:padding: 0 + +:::{grid-item-card} Time Series Data +:link: timeseries +:link-type: ref +:link-alt: Managing Time Series Data with CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`query_stats;1.3em` +::: + + +:::{grid-item-card} Machine Learning +:link: machine-learning +:link-type: ref +:link-alt: Machine Learning with CrateDB +:padding: 3 +:text-align: center +:class-card: sd-pt-3 +:class-body: sd-fs-1 +:class-title: sd-fs-5 + +{material-outlined}`model_training;1.3em` +::: + +:::: + ## Integrations @@ -186,42 +223,6 @@ frameworks. :::: -## Application Domains - -Learn how to apply CrateDB's features to optimally cover different -application and topic domains. - -::::{grid} 3 -:padding: 0 - -:::{grid-item-card} Time Series Data -:link: timeseries -:link-type: ref -:link-alt: Managing Time Series Data with CrateDB -:padding: 3 -:text-align: center -:class-card: sd-pt-3 -:class-body: sd-fs-1 -:class-title: sd-fs-5 - -{material-outlined}`query_stats;1.3em` -::: - - -:::{grid-item-card} Machine Learning -:link: machine-learning -:link-type: ref -:link-alt: Machine Learning with CrateDB -:padding: 3 -:text-align: center -:class-card: sd-pt-3 -:class-body: sd-fs-1 -:class-title: sd-fs-5 - -{material-outlined}`model_training;1.3em` -::: - -:::: ```{toctree} @@ -230,10 +231,10 @@ application and topic domains. install/index getting-started/index admin/index -integrate/index reference-architectures/index domain/index +integrate/index ``` From 70ebac5774d2590304accdec9b17858b98f500b7 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 23 Feb 2024 09:52:08 +0100 Subject: [PATCH 4/4] Chore: Fix cross references --- docs/integrate/bi/powerbi-desktop.rst | 2 +- docs/integrate/bi/powerbi-gateway.rst | 2 +- docs/reference-architectures/cratedb-azure-iot.rst | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/integrate/bi/powerbi-desktop.rst b/docs/integrate/bi/powerbi-desktop.rst index f8cc67a5..bec2a03f 100644 --- a/docs/integrate/bi/powerbi-desktop.rst +++ b/docs/integrate/bi/powerbi-desktop.rst @@ -1,4 +1,4 @@ -.. _cratedb-powerbi-desktop: +.. _powerbi-desktop: ========================================= Reports with CrateDB and Power BI Desktop diff --git a/docs/integrate/bi/powerbi-gateway.rst b/docs/integrate/bi/powerbi-gateway.rst index 30ab0f61..a07efe81 100644 --- a/docs/integrate/bi/powerbi-gateway.rst +++ b/docs/integrate/bi/powerbi-gateway.rst @@ -1,4 +1,4 @@ -.. _cratedb-powerbi-service: +.. _powerbi-service: =========================================== Real Time Reports with CrateDB and Power BI diff --git a/docs/reference-architectures/cratedb-azure-iot.rst b/docs/reference-architectures/cratedb-azure-iot.rst index a60e54cf..f3614cc0 100644 --- a/docs/reference-architectures/cratedb-azure-iot.rst +++ b/docs/reference-architectures/cratedb-azure-iot.rst @@ -22,8 +22,7 @@ Integrations This architecture makes use of the following integrations with CrateDB: -1. :ref:`cratedb-azure-functions` +1. :ref:`azure-functions` 2. :ref:`cratedb-r` -3. :ref:`cratedb-powerbi-desktop` or :ref:`cratedb-powerbi-service`, depending +3. :ref:`powerbi-desktop` or :ref:`powerbi-service`, depending on your requirements and use case. -