diff --git a/doc/user/assets/sass/_base.scss b/doc/user/assets/sass/_base.scss index 6c59c6f46e680..de6a57fbb83ea 100644 --- a/doc/user/assets/sass/_base.scss +++ b/doc/user/assets/sass/_base.scss @@ -11,10 +11,10 @@ $rem-scale: 0.585; :root { --nav-height: #{rem(6.5)}; // Changes to height requires changes to partials/toc.html since custom scroll logic implementation - --h1: #{rem(4.2)}; - --h2: #{rem(2.6)}; - --h3: #{rem(2.2)}; - --h4: #{rem(1.8)}; + --h1: #{rem(4.0)}; + --h2: #{rem(2.8)}; + --h3: #{rem(2.1)}; + --h4: #{rem(1.7)}; --h5: #{rem(1.6)}; --base: #{rem(1.6)}; --sm: #{rem(1.4)}; @@ -116,6 +116,10 @@ body.dark { --public-preview-border: #374e7c; --public-preview-after: #c0d3d8; --public-preview-gutter: #306dca; + + --tab-bg: var(--gray-lightest); + --tab-selected-bg: var(--tip-after); + --font-h4: var(--gray-lightest); } body.light { @@ -165,6 +169,10 @@ body.light { --public-preview-after: #c0d3d8; --public-preview-gutter: #186477; --public-preview-text: var(--black); + + --tab-bg: var(--gray-lighter); + --tab-selected-bg: var(--tip); + --font-h4: var(--gray-dark); } *, diff --git a/doc/user/assets/sass/_content.scss b/doc/user/assets/sass/_content.scss index 4a78c38bb0495..1e2c7a765c2f7 100644 --- a/doc/user/assets/sass/_content.scss +++ b/doc/user/assets/sass/_content.scss @@ -8,8 +8,6 @@ h3, h4, h5, h6 { - color: var(--important); - font-weight: 500; line-height: 1.2; code { @@ -24,6 +22,11 @@ h6 { } } +h1, h2, h3 { + font-weight: 500; + color: var(--important); +} + h1 { font-size: var(--h1); margin-bottom: var(--xx-small); @@ -36,11 +39,17 @@ h2 { h3 { font-size: var(--h3); + font-weight: 600; margin: var(--small) 0 var(--nano); } -.tab-pane h3 { - margin: 0; +div.tab-pane > h3 { + margin: var(--nano) 0; +} + +h4, h5, h6 { + font-weight: 600; + color: var(--font-h4); } h4 { @@ -48,6 +57,10 @@ h4 { margin: var(--small) 0 var(--nano); } +div.tab-pane > h4 { + margin: var(--nano) 0; +} + h5 { font-size: var(--h5); margin: var(--small) 0 var(--nano); @@ -251,8 +264,8 @@ p+p { &::before { content: ""; - width: 48px; - height: 48px; + width: 32px; + height: 32px; position: absolute; top: rem(2); background-size: contain; @@ -261,10 +274,16 @@ p+p { &.book, &.bulb, &.doc, + &.materialize, &.touch { padding-top: rem(8); } + &.materialize::before { + background-image: url("../images/materialize_favicon_32.png"); + margin-left: -10px; + } + &.book::before { background-image: url("../images/icon_book.png"); } @@ -283,6 +302,7 @@ p+p { .title { text-transform: uppercase; + font-size: 12px; font-weight: 500; color: var(--important); margin-bottom: var(--nano); @@ -546,8 +566,7 @@ p+p { } .annotation { - border: 4px solid var(--annotation-border); - background: var(--annotation-bg); + border: 1px solid var(--annotation-border); a { &:hover, @@ -752,7 +771,7 @@ p+p { .nav-tabs { list-style: none; padding: 0; - border-bottom: 1px solid #9c86e0; + display: flex; overflow-x: auto; // white-space: nowrap; @@ -763,7 +782,12 @@ p+p { padding: 0; position: relative; bottom: -1px; - background: var(--gray-lightest); + background: var(--bg); + border-top: 1px solid var(--highlight); + border-left: 1px solid var(--highlight); + border-right: 1px solid var(--highlight); + border-radius: 2px 2px 2px 2px; + background: var(--tab-bg); a { color: var(--body); @@ -788,13 +812,14 @@ p+p { } &.active { - background: var(--bg); + border-radius: 2px 2px 0 0; border: 1px solid #9c86e0; border-bottom-color: var(--bg); a { color: var(--highlight); + background: var(--tab-selected-bg); } } } @@ -807,6 +832,9 @@ p+p { &.active { display: block; + border-radius: 0 0 rem(1) rem(1); + border: 1px solid #9c86e0; + padding: rem(0.8) rem(0.5) rem(0.6) rem(0.6); } } @@ -1233,8 +1261,3 @@ sup { color: var(--link); } } - -// Remove the bottom border from the last row of a nested table -table table tr:last-child { - border-bottom: none; -} diff --git a/doc/user/assets/sass/_layout.scss b/doc/user/assets/sass/_layout.scss index e207b2c516950..fe7d0cbcfe7c8 100644 --- a/doc/user/assets/sass/_layout.scss +++ b/doc/user/assets/sass/_layout.scss @@ -199,7 +199,7 @@ table.inline-headings { display: block; cursor: pointer; position: relative; - padding: 3px 0 3px 15px; + padding: 3px 30px 3px 15px; margin-left: 5px; text-decoration: none; word-wrap: break-word; @@ -207,6 +207,7 @@ table.inline-headings { &.active { font-weight: 500; + background: var(--tip); } &:hover, @@ -225,10 +226,11 @@ table.inline-headings { svg { position: absolute; - left: -6px; + right: 6px; top: 50%; - margin-top: -18px; + transform: translateY(-50%); color: var(--divider-light); + transition: transform 0.2s ease-out; .darker { color: var(--sub); @@ -240,6 +242,10 @@ table.inline-headings { } } + li.open > a svg { + transform: translateY(-50%) rotate(90deg); + } + code { font-family: "Fira Code", Courier, monospace; } @@ -248,8 +254,7 @@ table.inline-headings { .level-4 a { - &:hover, - &.active { + &:hover { border-left: 2px solid var(--highlight); } } @@ -508,27 +513,6 @@ body.dark { fill: var(--gray-dark); } } - - li.level-2 { - li.level-3 { - ul { - - &:before, - &:after { - content: ""; - background-image: url("../images/level_three_transition_dark.svg"); - } - } - } - ul { - - &:before, - &:after { - content: ""; - background-image: url("../images/level_three_transition_dark.svg"); - } - } - } } .banner { diff --git a/doc/user/config.toml b/doc/user/config.toml index e07464d744cc1..62f664ddbc08c 100644 --- a/doc/user/config.toml +++ b/doc/user/config.toml @@ -20,7 +20,7 @@ publishDate = ["publishDate"] [[menu.main]] identifier = "get-started" name = "Get started" -weight = 5 +weight = 1 # # Connect sources @@ -67,7 +67,7 @@ weight = 12 [[menu.main]] identifier = "reference" name = "Reference" -weight = 15 +weight = 60 [[menu.main]] name = "SQL commands" @@ -80,16 +80,11 @@ weight = 100 # Releases # -[[menu.main]] -identifier = "releases-previews" -name = "Releases" -weight = 65 - [[menu.main]] identifier = "changelogs" name = "Changelog ↗️" -parent = "releases-previews" -weight = 20 +parent = "releases" +weight = 40 url = "https://materialize.com/changelog/" # @@ -99,10 +94,10 @@ url = "https://materialize.com/changelog/" [[menu.main]] identifier = "about" name = "About" -weight = 70 +weight = 90 [[menu.main]] -name = "Security overview" +name = "Materialize Cloud Security and Compliance" parent = "about" url = "https://materialize.com/security-overview" weight = 25 @@ -117,6 +112,16 @@ weight = 30 # allow , the old syntax no longer works unsafe = true +[markup] + [markup.highligh] + noClasses = false + style = "monokai" + +# Need to co-ordinate with toc.html + [markup.tableOfContents] + startLevel = 2 + endLevel = 4 + [[deployment.targets]] name = "production" url = "s3://materialize-website?region=us-east-1" diff --git a/doc/user/content/administration/billing.md b/doc/user/content/administration/billing.md index f86533440ae41..cb0bfebd79e4d 100644 --- a/doc/user/content/administration/billing.md +++ b/doc/user/content/administration/billing.md @@ -93,7 +93,7 @@ Accessing usage and billing information in Materialize requires **administrator** privileges. {{}} -From the [Materialize console](https://console.materialize.com/) (`Admin` > +From the [Materialize console](/console/) (`Admin` > `Usage & Billing`), administrators can access their invoice. The invoice provides Compute and Storage usage and cost information. @@ -101,7 +101,7 @@ provides Compute and Storage usage and cost information. Materialize Cloud administrators can sign up for an [On Demand plan](https://materialize.com/pdfs/on-demand-terms.pdf) from the billing section -of the [Materialize console](https://console.materialize.com/). Pricing is +of the [Materialize console](/console/). Pricing is usage-based and is billed on a monthly basis. Invoices will be sent to the account email and paid via the card on file on the first of the month. If you have questions about billing or are interested in converting to an annual diff --git a/doc/user/content/console/_index.md b/doc/user/content/console/_index.md index 4de39eecf223b..16e268d4c2b91 100644 --- a/doc/user/content/console/_index.md +++ b/doc/user/content/console/_index.md @@ -5,11 +5,12 @@ disable_list: true disable_toc: true menus: main: - weight: 5 + parent: "get-started" + weight: 25 identifier: 'console' --- -The [Materialize Console](https://console.materialize.com/) is a graphical user +The [Materialize Console](/console/) is a graphical user interface for working with Materialize. From the Console, you can create and manage your clusters and sources, issue SQL queries, explore your objects, and view billing information. diff --git a/doc/user/content/console/admin.md b/doc/user/content/console/admin.md index 00125c52b04cf..cbdc3dbc97323 100644 --- a/doc/user/content/console/admin.md +++ b/doc/user/content/console/admin.md @@ -9,7 +9,7 @@ menu: identifier: console-admin --- -The [Materialize Console](https://console.materialize.com/) provides an +The [Materialize Console](/console/) provides an **Admin** section where you can manage client credentials and, for administrators, review your usage and billing information. diff --git a/doc/user/content/console/clusters.md b/doc/user/content/console/clusters.md index a89d1b7671b4e..d030a09a14de7 100644 --- a/doc/user/content/console/clusters.md +++ b/doc/user/content/console/clusters.md @@ -9,7 +9,7 @@ menu: identifier: console-clusters --- -The [Materialize Console](https://console.materialize.com/) provides a +The [Materialize Console](/console/) provides a **Clusters** section where you can manage your clusters. ![Image of Clusters page](/images/console/console-clusters.png "Clusters page diff --git a/doc/user/content/console/create-new.md b/doc/user/content/console/create-new.md index 7f57314bb87ae..face3d51e3bb2 100644 --- a/doc/user/content/console/create-new.md +++ b/doc/user/content/console/create-new.md @@ -17,7 +17,7 @@ Materialize to read data from"), and application passwords. ![Image of the Create New Cluster flow](/images/console/console-create-new/postgresql/create-new-cluster-flow.png "Create New Cluster flow") -From the [Materialize console](https://console.materialize.com/): +From the [Materialize console](/console/): 1. Click **+ Create New** and select **Cluster** to open the **New cluster** screen. @@ -55,7 +55,7 @@ From the [Materialize console](https://console.materialize.com/): PostgreSQL](/images/console/console-create-new/postgresql/create-new-source-start.png "Create New Source start for PostgreSQL") -From the [Materialize console](https://console.materialize.com/): +From the [Materialize console](/console/): 1. Click **+ Create New** and select **Source** to open the **New source** screen. diff --git a/doc/user/content/console/data.md b/doc/user/content/console/data.md index fabd86a36282d..665f873e8e6c4 100644 --- a/doc/user/content/console/data.md +++ b/doc/user/content/console/data.md @@ -9,7 +9,7 @@ menu: identifier: console-data --- -Under **Data**, the [Materialize Console](https://console.materialize.com/) +Under **Data**, the [Materialize Console](/console/) provides a database object explorer. ![Image of the Materialize Console Database Object diff --git a/doc/user/content/console/integrations.md b/doc/user/content/console/integrations.md index d605a0a65ff00..08911ac3e3be2 100644 --- a/doc/user/content/console/integrations.md +++ b/doc/user/content/console/integrations.md @@ -9,7 +9,7 @@ menu: identifier: console-integrations --- -The [Materialize Console](https://console.materialize.com/) provides an +The [Materialize Console](/console/) provides an **Integrations** page that lists the supported 3rd party integrations, specifying: diff --git a/doc/user/content/console/monitoring.md b/doc/user/content/console/monitoring.md index bc1e259c742eb..84b5b13c4733c 100644 --- a/doc/user/content/console/monitoring.md +++ b/doc/user/content/console/monitoring.md @@ -9,7 +9,7 @@ menu: identifier: console-monitoring --- -The [Materialize Console](https://console.materialize.com/) provides a +The [Materialize Console](/console/) provides a **Monitoring** section where you can review the health of your environment and access your query history. diff --git a/doc/user/content/console/sql-shell.md b/doc/user/content/console/sql-shell.md index b70593475b686..17fa3b5473abb 100644 --- a/doc/user/content/console/sql-shell.md +++ b/doc/user/content/console/sql-shell.md @@ -9,7 +9,7 @@ menu: identifier: console-sql-shell --- -The [Materialize Console](https://console.materialize.com/) provides a **SQL +The [Materialize Console](/console/) provides a **SQL Shell**, where you can issue your queries. Materialize follows the SQL standard (SQL-92) implementation, and strives for compatibility with the PostgreSQL dialect. If your query takes too long to complete, the SQL Shell provides diff --git a/doc/user/content/console/user-profile.md b/doc/user/content/console/user-profile.md index e182a522a0bc8..8c8ca7560beb5 100644 --- a/doc/user/content/console/user-profile.md +++ b/doc/user/content/console/user-profile.md @@ -10,7 +10,7 @@ menu: --- You can manage your user profile and account settings from the [Materialize -Console](https://console.materialize.com/). +Console](/console/). ![Image of the user profile menu](/images/console/console-user-profile.png "User profile menu") diff --git a/doc/user/content/customer-responsibilities.md b/doc/user/content/customer-responsibilities.md index f4ee4171f2918..9aff0351e8172 100644 --- a/doc/user/content/customer-responsibilities.md +++ b/doc/user/content/customer-responsibilities.md @@ -1,24 +1,24 @@ --- -title: "Customer responsibility model" -description: "Details about the Materialize's customer responsibility model." +title: "Materialize Cloud Customer responsibility model" +description: "Details about the Materialize Cloud's customer responsibility model." menu: main: parent: 'about' weight: 10 --- -The Materialize platform is built with a **shared responsibility model** to -ensure the highest levels of data integrity, availability, and resilience. This -page outlines the specific responsibilities of customers to manage their data -and connectivity effectively. Understanding and fulfilling these +The Materialize Cloud platform is built with a **shared responsibility model** +to ensure the highest levels of data integrity, availability, and resilience. +This page outlines the specific responsibilities of customers to manage their +data and connectivity effectively. Understanding and fulfilling these responsibilities is critical to leveraging the full potential of the platform. ### Backup -As part of the Materialize shared responsibility model, customers are tasked -with maintaining backups of their original data. While Materialize provides -robust data processing capabilities, ensuring that you have a comprehensive -backup strategy is crucial. This includes: +As part of the Materialize Cloud's shared responsibility model, customers are +tasked with maintaining backups of their original data. While Materialize +provides robust data processing capabilities, ensuring that you have a +comprehensive backup strategy is crucial. This includes: - **Regular backups**. Implementing regular backup schedules that align with your data recovery and business continuity plans. diff --git a/doc/user/content/free-trial-faqs.md b/doc/user/content/free-trial-faqs.md index abead4dae39c1..3e2888a813990 100644 --- a/doc/user/content/free-trial-faqs.md +++ b/doc/user/content/free-trial-faqs.md @@ -1,15 +1,15 @@ --- title: "Free trial FAQs" -description: "Answers to frequently asked questions about Materialize free trials" +description: "Answers to frequently asked questions about Materialize Cloud free trials" menu: main: parent: "about" weight: 20 --- -When you [sign up for Materialize](https://materialize.com/register/), you get a -free trial account so you can explore the product and start building! This page -answers some frequently asked questions about free trials. +When you [sign up for Materialize Cloud](https://materialize.com/register/), you +get a free trial account so you can explore the product and start building! This +page answers some frequently asked questions about free trials. {{< tip >}} For help getting started with your data or other questions about Materialize, you can schedule a [free guided trial](https://materialize.com/demo/?utm_campaign=General&utm_source=documentation). @@ -42,7 +42,7 @@ is under the rate limit of 4 credits per hour. To continue using Materialize, you can upgrade to a paid, [On Demand plan](https://materialize.com/pdfs/on-demand-terms.pdf) from the billing section -of the [Materialize console](https://console.materialize.com/). Otherwise, +of the [Materialize console](/console/). Otherwise, Materialize will delete your resources and data at the end of the trial period. ## How do I monitor my credit consumption rate? diff --git a/doc/user/content/get-started/install-materialize-emulator.md b/doc/user/content/get-started/install-materialize-emulator.md index bb81e1e3458a2..412d1fabccdd0 100644 --- a/doc/user/content/get-started/install-materialize-emulator.md +++ b/doc/user/content/get-started/install-materialize-emulator.md @@ -4,7 +4,7 @@ description: "The Materialize Emulator is an all-in-one Docker image available o menu: main: parent: "get-started" - weight: 16 + weight: 18 name: "Download and run Materialize Emulator" --- diff --git a/doc/user/content/get-started/install.md b/doc/user/content/get-started/install.md new file mode 100644 index 0000000000000..8676076022667 --- /dev/null +++ b/doc/user/content/get-started/install.md @@ -0,0 +1,68 @@ +--- +title: "Install self-managed Materialize" +description: "Install self-managed Materialize." +disable_list: true +menu: + main: + parent: get-started + identifier: "install" + weight: 17 + +--- + +You can install self-managed Materialize on a Kubernetes cluster running locally +or on a cloud provider. Self-managed Materialize requires: + +{{% self-managed/materialize-components-list %}} + +## Install locally + +{{< multilinkbox >}} +{{< linkbox title="Using Docker/kind" icon="materialize">}} +[Install locally on kind](/self-managed/installation/install-on-local-kind/) +{{}} +{{< linkbox title="Using Docker/minikube" icon="materialize">}} +[Install locally on minikube](/self-managed/installation/install-on-local-minikube/) +{{}} +{{}} + +## Install on cloud provider + +{{< multilinkbox >}} + +{{< linkbox title="[Install on AWS](/self-managed/installation/install-on-aws/)" icon="materialize">}} + +[Deploy Materialize to AWS Elastic Kubernetes Service (EKS)](/self-managed/installation/install-on-aws/) + +{{}} + + +{{< linkbox title="[Install on Azure](/self-managed/installation/install-on-azure/)" icon="materialize">}} + +[Deploy Materialize to Azure Kubernetes Service (AKS)](/self-managed/installation/install-on-azure/) + +{{}} + +{{< linkbox icon="materialize" title="[Install on GCP](/self-managed/installation/install-on-gcp/)" >}} + +[Deploy Materialize to Google Kubernetes Engine (GKE)](/self-managed/installation/install-on-gcp/) +{{}} + +{{}} + +See also: + +- [AWS Deployment + guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/#recommended-instance-types) + +- [GCP Deployment + guidelines](/self-managed/installation/install-on-gcp/appendix-deployment-guidelines/#recommended-instance-types) + +- [Azure Deployment + guidelines](/self-managed/installation/install-on-azure/appendix-deployment-guidelines/#recommended-instance-types) + +{{< callout >}} + +{{< self-managed/also-available >}} + +{{}} diff --git a/doc/user/content/get-started/quickstart.md b/doc/user/content/get-started/quickstart.md index 051a36771d222..bf01c85ac78ce 100644 --- a/doc/user/content/get-started/quickstart.md +++ b/doc/user/content/get-started/quickstart.md @@ -60,7 +60,7 @@ Emulator does not provide the full experience of using Materialize. ## Step 0. Open the SQL Shell - If you have a Materialize account, navigate to the [Materialize - Console](https://console.materialize.com/) and sign in. By default, you should + Console](/console/) and sign in. By default, you should be in the SQL Shell. If you're already signed in, you can access the SQL Shell in the left-hand menu. - If you are using the Materialize Emulator, open the Materialize Console in @@ -603,7 +603,7 @@ creating indexes, see [Index Best Practices](/concepts/indexes/#best-practices). To get started ingesting your own data from an external system like Kafka, MySQL or PostgreSQL, check the documentation for [sources](/sql/create-source/), and -navigate to **Data** > **Sources** > **New source** in the [Materialize Console](https://console.materialize.com/) +navigate to **Data** > **Sources** > **New source** in the [Materialize Console](/console/) to create your first source. For help getting started with your data or other questions about Materialize, diff --git a/doc/user/content/ingest-data/cdc-cockroachdb.md b/doc/user/content/ingest-data/cdc-cockroachdb.md index 66b0f15ea372b..b707e1a0a49f3 100644 --- a/doc/user/content/ingest-data/cdc-cockroachdb.md +++ b/doc/user/content/ingest-data/cdc-cockroachdb.md @@ -78,7 +78,7 @@ refer to the [CockroachDB documentation](https://www.cockroachlabs.com/docs/stab ## B. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your Kafka @@ -102,7 +102,7 @@ authentication and networking configurations, so refer to the [`CREATE CONNECTION`](/sql/create-connection/#kafka) documentation for further guidance. -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the credentials to connect to your Kafka broker and, optionally, schema registry: diff --git a/doc/user/content/ingest-data/kafka/amazon-msk.md b/doc/user/content/ingest-data/kafka/amazon-msk.md index ffe55ed706c1f..933908f8b36cb 100644 --- a/doc/user/content/ingest-data/kafka/amazon-msk.md +++ b/doc/user/content/ingest-data/kafka/amazon-msk.md @@ -208,7 +208,7 @@ The process to connect Materialize to Amazon MSK consists of the following steps c. Copy the url under **Private endpoint** and against **SASL/SCRAM**. This will be your `` going forward. - d. Connect to Materialize using the [SQL Shell](https://console.materialize.com/), + d. Connect to Materialize using the [SQL Shell](/console/), or your preferred SQL client. e. Create a connection using the command below. The broker URL is what you copied in step c of this subsection. The `` is the name of the topic you created in Step 4. The `` and `` is from _Store a new secret_ under Step 2. diff --git a/doc/user/content/ingest-data/kafka/confluent-cloud.md b/doc/user/content/ingest-data/kafka/confluent-cloud.md index 0fab7933082ff..e15eb5beb75c3 100644 --- a/doc/user/content/ingest-data/kafka/confluent-cloud.md +++ b/doc/user/content/ingest-data/kafka/confluent-cloud.md @@ -84,7 +84,7 @@ of the following steps: c. Copy the URL under **Bootstrap server**. This will be your `` going forward. - d. Connect to Materialize using the [SQL Shell](https://console.materialize.com/), + d. Connect to Materialize using the [SQL Shell](/console/), or your preferred SQL client. e. Run the following command. Replace `` with whatever you diff --git a/doc/user/content/ingest-data/kafka/kafka-self-hosted.md b/doc/user/content/ingest-data/kafka/kafka-self-hosted.md index e990bfbaf4a2f..38c915ee07a18 100644 --- a/doc/user/content/ingest-data/kafka/kafka-self-hosted.md +++ b/doc/user/content/ingest-data/kafka/kafka-self-hosted.md @@ -84,7 +84,7 @@ CREATE CONNECTION kafka_connection TO KAFKA ( {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: diff --git a/doc/user/content/ingest-data/mysql/amazon-aurora.md b/doc/user/content/ingest-data/mysql/amazon-aurora.md index a5d7c8159a582..4358d5ccc9e61 100644 --- a/doc/user/content/ingest-data/mysql/amazon-aurora.md +++ b/doc/user/content/ingest-data/mysql/amazon-aurora.md @@ -92,6 +92,11 @@ can skip this step**. For production scenarios, we recommend configuring one of the network security options below. {{< /note >}} + +{{< tabs >}} +{{< tab "Materialize Cloud" >}} + + There are various ways to configure your database's network to allow Materialize to connect: @@ -110,7 +115,7 @@ to connect: {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -253,7 +258,7 @@ configuration of resources for an SSH tunnel. For more details, see the 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -280,9 +285,71 @@ configuration of resources for an SSH tunnel. For more details, see the {{< /tabs >}} +{{< /tab >}} +{{< tab "Materialize Self-Managed" >}} + + +There are various ways to configure your database’s network to allow Materialize +to connect: + +{{< tabs >}} + +{{< tab "Allow Materialize IPs">}} + +1. [Add an inbound rule to your Aurora security group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Overview.RDSSecurityGroups.html) + to allow traffic from Materialize IPs. + + In each rule: + + - Set **Type** to **MySQL**. + - Set **Source** to the IP address in CIDR notation. + +{{< /tab >}} +{{< tab "Use an SSH tunnel">}} + +To create an SSH tunnel from Materialize to your database, you launch an +instance to serve as an SSH bastion host, configure the bastion host to allow +traffic only from Materialize, and then configure your database's private +network to allow traffic from the bastion host. + +{{< note >}} +Materialize provides a Terraform module that automates the creation and +configuration of resources for an SSH tunnel. For more details, see the +[Terraform module repository](https://github.com/MaterializeInc/terraform-aws-ec2-ssh-bastion). +{{}} + +1. [Launch an EC2 instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/LaunchingAndUsingInstances.html) + to serve as your SSH bastion host. + + - Make sure the instance is publicly accessible and in the same VPC as your + Amazon Aurora MySQL instance. + + - Add a key pair and note the username. You'll use this username when + connecting Materialize to your bastion host. + + **Warning:** Auto-assigned public IP addresses can change in [certain cases](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-public-addresses). + For this reason, it's best to associate an [elastic IP address](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html#ip-addressing-eips) + to your bastion host. + +1. Configure the SSH bastion host to allow traffic only from Materialize. + +1. In the security group of your RDS instance, [add an inbound rule](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Overview.RDSSecurityGroups.html) + to allow traffic from the SSH bastion host. + + - Set **Type** to **All TCP**. + - Set **Source** to **Custom** and select the bastion host's security + group. + +{{< /tab >}} + +{{< /tabs >}} +{{< /tab >}} + +{{< /tabs >}} + ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Optional) Create a source cluster {{< note >}} If you are prototyping and already have a cluster to host your MySQL @@ -293,14 +360,10 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% mysql-direct/create-a-cluster %}} -### 2. Start ingesting data - -[//]: # "TODO(morsapaes) MySQL connections support multiple SSL modes. We should -adapt to that, rather than just state SSL MODE REQUIRED." +### 2. Create a connection -Now that you've configured your database network, you can connect Materialize to -your MySQL database and start ingesting data. The exact steps depend on your -networking configuration, so start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} @@ -318,15 +381,23 @@ networking configuration, so start by selecting the relevant option. {{< /tabs >}} +### 3. Start ingesting data + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="schema-changes" %}} + [//]: # "TODO(morsapaes) Replace these Step 6. and 7. with guidance using the new progress metrics in mz_source_statistics + console monitoring, when available (also for PostgreSQL)." -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% mysql-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% mysql-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/mysql/amazon-rds.md b/doc/user/content/ingest-data/mysql/amazon-rds.md index e4fce5b23af8c..c683907949589 100644 --- a/doc/user/content/ingest-data/mysql/amazon-rds.md +++ b/doc/user/content/ingest-data/mysql/amazon-rds.md @@ -123,7 +123,7 @@ to connect: {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -265,7 +265,7 @@ configuration of resources for an SSH tunnel. For more details, see the 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -293,7 +293,7 @@ configuration of resources for an SSH tunnel. For more details, see the ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your MySQL @@ -304,15 +304,13 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% mysql-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection [//]: # "TODO(morsapaes) MySQL connections support multiple SSL modes. We should adapt to that, rather than just state SSL MODE REQUIRED." -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your MySQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} @@ -330,15 +328,23 @@ start by selecting the relevant option. {{< /tabs >}} +### 3. Start ingesting data + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="schema-changes" %}} + [//]: # "TODO(morsapaes) Replace these Step 6. and 7. with guidance using the new progress metrics in mz_source_statistics + console monitoring, when available (also for PostgreSQL)." -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% mysql-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% mysql-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/mysql/azure-db.md b/doc/user/content/ingest-data/mysql/azure-db.md index 7a6df1ca00645..990392049ca03 100644 --- a/doc/user/content/ingest-data/mysql/azure-db.md +++ b/doc/user/content/ingest-data/mysql/azure-db.md @@ -65,7 +65,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -97,7 +97,7 @@ to serve as your SSH bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -117,7 +117,7 @@ to serve as your SSH bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your MySQL @@ -128,14 +128,13 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% mysql-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection [//]: # "TODO(morsapaes) MySQL connections support multiple SSL modes. We should adapt to that, rather than just state SSL MODE REQUIRED." -Now that you've configured your database network, you can connect Materialize to -your MySQL database and start ingesting data. The exact steps depend on your -networking configuration, so start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} @@ -149,15 +148,19 @@ networking configuration, so start by selecting the relevant option. {{< /tabs >}} -[//]: # "TODO(morsapaes) Replace these Step 6. and 7. with guidance using the -new progress metrics in mz_source_statistics + console monitoring, when -available(also for PostgreSQL)." +### 3. Start ingesting data -### 3. Monitor the ingestion status +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="schema-changes" %}} + +### 4. Monitor the ingestion status {{% mysql-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% mysql-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/mysql/google-cloud-sql.md b/doc/user/content/ingest-data/mysql/google-cloud-sql.md index 0961f9d68de3a..af3ca77edab65 100644 --- a/doc/user/content/ingest-data/mysql/google-cloud-sql.md +++ b/doc/user/content/ingest-data/mysql/google-cloud-sql.md @@ -59,7 +59,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -91,7 +91,7 @@ network to allow traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -111,7 +111,7 @@ bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your MySQL @@ -122,14 +122,13 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% mysql-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection [//]: # "TODO(morsapaes) MySQL connections support multiple SSL modes. We should adapt to that, rather than just state SSL MODE REQUIRED." -Now that you've configured your database network, you can connect Materialize to -your MySQL database and start ingesting data. The exact steps depend on your -networking configuration, so start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} @@ -143,11 +142,19 @@ networking configuration, so start by selecting the relevant option. {{< /tabs >}} -### 3. Monitor the ingestion status +### 3. Start ingesting data + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="schema-changes" %}} + +### 4. Monitor the ingestion status {{% mysql-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% mysql-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/mysql/self-hosted.md b/doc/user/content/ingest-data/mysql/self-hosted.md index ecb1da5648c41..68254866672bc 100644 --- a/doc/user/content/ingest-data/mysql/self-hosted.md +++ b/doc/user/content/ingest-data/mysql/self-hosted.md @@ -60,7 +60,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -91,7 +91,7 @@ traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -111,7 +111,7 @@ traffic from the bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your MySQL @@ -122,11 +122,10 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% mysql-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network, you can connect Materialize to -your MySQL database and start ingesting data. The exact steps depend on your -networking configuration, so start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} @@ -140,16 +139,19 @@ networking configuration, so start by selecting the relevant option. {{< /tabs >}} +### 3. Start ingesting data -[//]: # "TODO(morsapaes) Replace these Step 6. and 7. with guidance using the -new progress metrics in mz_source_statistics + console monitoring, when -available(also for PostgreSQL)." +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source" %}} -### 3. Monitor the ingestion status +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/mysql/create_source_cloud" example="schema-changes" %}} + +### 4. Monitor the ingestion status {{% mysql-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% mysql-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/alloydb.md b/doc/user/content/ingest-data/postgres/alloydb.md index 8ca3a49c508c9..c1b9ef9c3fed9 100644 --- a/doc/user/content/ingest-data/postgres/alloydb.md +++ b/doc/user/content/ingest-data/postgres/alloydb.md @@ -67,7 +67,7 @@ Materialize with AlloyDB: {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), +1. In the [Materialize console's SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -101,7 +101,7 @@ network to allow traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. 1. In the [Materialize console's SQL - Shell](https://console.materialize.com/), or your preferred SQL client + Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -121,7 +121,7 @@ network to allow traffic from the bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -132,28 +132,80 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -With the network configured and an ingestion pipeline in place, connect -Materialize to your AlloyDB instance and begin the data ingestion process. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} {{< tab "Allow Materialize IPs">}} -{{% postgres-direct/ingesting-data/allow-materialize-ips %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-general" indent="true" %}} + {{< /tab >}} {{< tab "Use an SSH tunnel">}} -{{% postgres-direct/ingesting-data/use-ssh-tunnel %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-general" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-secret" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-general" indent="true" %}} {{< /tab >}} {{< /tabs >}} -### 3. Monitor the ingestion status +### 3. Start ingesting data + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} + +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/amazon-aurora.md b/doc/user/content/ingest-data/postgres/amazon-aurora.md index 6eb8975dce4cb..0656efc1eb8bf 100644 --- a/doc/user/content/ingest-data/postgres/amazon-aurora.md +++ b/doc/user/content/ingest-data/postgres/amazon-aurora.md @@ -77,7 +77,7 @@ to connect: {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/) or your preferred SQL +1. In the [SQL Shell](/console/) or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -220,7 +220,7 @@ configuration of resources for an SSH tunnel. For more details, see the 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -249,7 +249,7 @@ configuration of resources for an SSH tunnel. For more details, see the ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -258,294 +258,122 @@ scenarios, we recommend separating your workloads into multiple clusters for [resource isolation](/sql/create-cluster/#resource-isolation). {{< /note >}} - {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your PostgreSQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - SECRET`](/sql/create-secret/) command to securely store the password for the - `materialize` PostgreSQL user you created - [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a - connection object with access and authentication details for Materialize to - use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - SSL MODE 'require', - DATABASE '' - ); - ``` - - - Replace `` with the **Writer** endpoint for your Aurora database. To - find the endpoint, select your database in the AWS Management Console, - then click the **Connectivity & security** tab and look for the endpoint - with type **Writer**. - -
- WARNING! - You must use the Writer endpoint for the database. Using a Reader endpoint will not work. -
+1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true" %}} -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your Aurora instance and start ingesting data from the publication you - created [earlier](#2-create-a-publication-and-a-replication-user). - - ```mzsql - CREATE SOURCE mz_source - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` - - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - -1. After source creation, you can handle upstream [schema changes](/sql/create-source/postgres/#schema-changes) - for specific replicated tables using the [`ALTER SOURCE...ADD SUBSOURCE`](/sql/alter-source/#context) - and [`DROP SOURCE`](/sql/alter-source/#dropping-subsources) syntax. + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-aurora" indent="true" %}} {{< /tab >}} {{< tab "Use AWS PrivateLink">}} -1. In the [SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - CONNECTION`](/sql/create-connection/#aws-privatelink) command to create an - AWS PrivateLink connection: - - ```mzsql - CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( - SERVICE NAME 'com.amazonaws.vpce.us-east-1.vpce-svc-0356210a8a432d9e9', - AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4') - ); - ``` - - - Replace the `SERVICE NAME` value with the service name you noted [earlier](#b-optional-configure-network-security). - - - Replace the `AVAILABILITY ZONES` list with the IDs of the availability - zones in your AWS account. - - To find your availability zone IDs, select your database in the RDS - Console and click the subnets under **Connectivity & security**. For each - subnet, look for **Availability Zone ID** (e.g., `use1-az6`), - not **Availability Zone** (e.g., `us-east-1d`). - -1. Retrieve the AWS principal for the AWS PrivateLink connection you just created: - - ```mzsql - SELECT principal - FROM mz_aws_privatelink_connections plc - JOIN mz_connections c ON plc.id = c.id - WHERE c.name = 'privatelink_svc'; - ``` -

- - ``` - principal - --------------------------------------------------------------------------- - arn:aws:iam::664411391173:role/mz_20273b7c-2bbe-42b8-8c36-8cc179e9bbc3_u1 - ``` - -1. Update your VPC endpoint service to [accept connections from the AWS principal](https://docs.aws.amazon.com/vpc/latest/privatelink/add-endpoint-service-permissions.html). - -1. If your AWS PrivateLink service is configured to require acceptance of - connection requests, [manually approve the connection request from Materialize](https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html#accept-reject-connection-requests). - - **Note:** It can take some time for the connection request to show up. Do - not move on to the next step until you've approved the connection. - -1. Validate the AWS PrivateLink connection you created using the - [`VALIDATE CONNECTION`](/sql/validate-connection) command: - - ```mzsql - VALIDATE CONNECTION privatelink_svc; - ``` - - If no validation error is returned, move to the next step. - -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the - password for the `materialize` PostgreSQL user you created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create -another connection object, this time with database access and authentication -details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - AWS PRIVATELINK privatelink_svc - ); - ``` - - - Replace `` with your Aurora endpoint. To find your Aurora endpoint, - select your database in the AWS Management Console, and look - under **Connectivity & security**. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. - -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your Aurora instance via AWS PrivateLink and start ingesting data from the - publication you created - [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SOURCE mz_source - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` - - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-aurora" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-options-aurora" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="get-principal-privatelink-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="get-principal-privatelink-connection-results" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="update-vpc-endpoint" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="approve-connection-request" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="validate-connection" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-secret" indent="true" %}} +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection" indent="true" %}} + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection-options-aurora" indent="true" %}} {{< /tab >}} {{< tab "Use an SSH tunnel">}} -1. In the [SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - CONNECTION`](/sql/create-connection/#ssh-tunnel) command to create an SSH - tunnel connection: - - ```mzsql - CREATE CONNECTION ssh_connection TO SSH TUNNEL ( - HOST '', - PORT , - USER '' - ); - ``` - - - Replace `` and ` with the public IP - address and port of the SSH bastion host you created [earlier](#b-optional-configure-network-security). - - - Replace `` with the username for the key pair you - created for your SSH bastion host. - -1. Get Materialize's public keys for the SSH tunnel connection you just - created: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true"%}} - ```mzsql - SELECT - mz_connections.name, - mz_ssh_tunnel_connections.* - FROM - mz_connections - JOIN - mz_ssh_tunnel_connections USING(id) - WHERE - mz_connections.name = 'ssh_connection'; - ``` - -1. Log in to your SSH bastion host and add Materialize's public keys to the - `authorized_keys` file, for example: - - ```sh - # Command for Linux - echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys - echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" %}} -1. Back in the SQL client connected to Materialize, validate the SSH tunnel - connection you created using the [`VALIDATE CONNECTION`](/sql/validate-connection) - command: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-aurora-rds-self-hosted" indent="true"%}} - ```mzsql - VALIDATE CONNECTION ssh_connection; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true"%}} - If no validation error is returned, move to the next step. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the -password for the `materialize` PostgreSQL user you created [earlier](#2-create-a-publication-and-a-replication-user): +1. {{% include-example file="examples/ingest_data/postgres/create_connection_ssh_cloud" example="create-secret" indent="true"%}} - ```mzsql - CREATE SECRET pgpass AS ''; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true"%}} -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-aurora" indent="true"%}} - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - SSH TUNNEL ssh_connection - ); - ``` +{{< /tab >}} - - Replace `` with your Aurora endpoint. To find your Aurora endpoint, - select your database in the AWS Management Console, and look - under **Connectivity & security**. +{{< /tabs >}} - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +### 3. Start ingesting data -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your Aurora instance and start ingesting data from the publication you - created [earlier](#2-create-a-publication-and-a-replication-user): +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} - ```mzsql - CREATE SOURCE mz_source - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - -{{< /tab >}} - -{{< /tabs >}} +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/amazon-rds.md b/doc/user/content/ingest-data/postgres/amazon-rds.md index 68da5d47ee910..7d2623614930a 100644 --- a/doc/user/content/ingest-data/postgres/amazon-rds.md +++ b/doc/user/content/ingest-data/postgres/amazon-rds.md @@ -125,7 +125,7 @@ to connect: {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -268,7 +268,7 @@ configuration of resources for an SSH tunnel. For more details, see the 1. Configure the SSH bastion host to allow traffic only from Materialize. 1. In the [Materialize console's SQL - Shell](https://console.materialize.com/), or your preferred SQL client + Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -296,7 +296,7 @@ configuration of resources for an SSH tunnel. For more details, see the ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -307,314 +307,135 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your PostgreSQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} - {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or - your preferred SQL client connected to Materialize, use the [`CREATE - SECRET`](/sql/create-secret/) command to securely store the password for the - `materialize` PostgreSQL user you created - [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a - connection object with access and authentication details for Materialize to - use: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true"%}} - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - SSL MODE 'require', - DATABASE '' - ); - ``` - - - Replace `` with your RDS endpoint. To find your RDS endpoint, select - your database in the RDS Console, and look under **Connectivity & - security**. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. - -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your RDS instance and start ingesting data from the publication you - created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` - - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - -1. After source creation, you can handle upstream [schema changes](/sql/create-source/postgres/#schema-changes) - for specific replicated tables using the [`ALTER SOURCE...ADD SUBSOURCE`](/sql/alter-source/#context) - and [`DROP SOURCE`](/sql/alter-source/#dropping-subsources) syntax. + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-rds" indent="true"%}} {{< /tab >}} {{< tab "Use AWS PrivateLink">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL - client connected to Materialize, use the [`CREATE CONNECTION`](/sql/create-connection/#aws-privatelink) - command to create an **in-region** or **cross-region** AWS PrivateLink - connection. - - ↕️ **In-region connections** - - To connect to an AWS PrivateLink endpoint service in the **same region** as your - Materialize environment: - - ```mzsql - CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( - SERVICE NAME 'com.amazonaws.vpce..vpce-svc-', - AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4') - ); - ``` - - - Replace the `SERVICE NAME` value with the service name you noted [earlier](#b-optional-configure-network-security). - - - Replace the `AVAILABILITY ZONES` list with the IDs of the availability - zones in your AWS account. For in-region connections the availability - zones of the NLB and the consumer VPC **must match**. - - To find your availability zone IDs, select your database in the RDS - Console and click the subnets under **Connectivity & security**. For each - subnet, look for **Availability Zone ID** (e.g., `use1-az6`), - not **Availability Zone** (e.g., `us-east-1d`). - - ↔️ **Cross-region connections** - - To connect to an AWS PrivateLink endpoint service in a **different region** to - the one where your Materialize environment is deployed: - - ```mzsql - CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( - SERVICE NAME 'com.amazonaws.vpce.us-west-1.vpce-svc-', - -- For now, the AVAILABILITY ZONES clause **is** required, but will be - -- made optional in a future release. - AVAILABILITY ZONES () - ); - ``` - - - Replace the `SERVICE NAME` value with the service name you noted [earlier](#b-optional-configure-network-security). - - - The service name region refers to where the endpoint service was created. - You **do not need** to specify `AVAILABILITY ZONES` manually — these will - be optimally auto-assigned when none are provided. - -1. Retrieve the AWS principal for the AWS PrivateLink connection you just - created: - - ```mzsql - SELECT principal - FROM mz_aws_privatelink_connections plc - JOIN mz_connections c ON plc.id = c.id - WHERE c.name = 'privatelink_svc'; - ``` -

- - ``` - principal - --------------------------------------------------------------------------- - arn:aws:iam::664411391173:role/mz_20273b7c-2bbe-42b8-8c36-8cc179e9bbc3_u1 - ``` - -1. Update your VPC endpoint service to [accept connections from the AWS principal](https://docs.aws.amazon.com/vpc/latest/privatelink/add-endpoint-service-permissions.html). - -1. If your AWS PrivateLink service is configured to require acceptance of - connection requests, [manually approve the connection request from Materialize](https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html#accept-reject-connection-requests). - - **Note:** It can take some time for the connection request to show up. Do - not move on to the next step until you've approved the connection. - -1. Validate the AWS PrivateLink connection you created using the - [`VALIDATE CONNECTION`](/sql/validate-connection) command: - - ```mzsql - VALIDATE CONNECTION privatelink_svc; - ``` - - If no validation error is returned, move to the next step. - -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the - password for the `materialize` PostgreSQL user you created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - AWS PRIVATELINK privatelink_svc - ); - ``` - - - Replace `` with your RDS endpoint. To find your RDS endpoint, select - your database in the RDS Console, and look under **Connectivity & - security**. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. - -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your RDS instance via AWS PrivateLink and start ingesting data from the - publication you created - [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` - - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-rds-intro" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-rds-in-region" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-rds-in-region-options" indent="true" + %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-rds-cross-region" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-privatelink-connection-rds-cross-region-options" + indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="get-principal-privatelink-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="get-principal-privatelink-connection-results" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="update-vpc-endpoint" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="approve-connection-request" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="validate-connection" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-secret" indent="true" %}} +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection" indent="true" %}} + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection-options-rds" indent="true" %}} {{< /tab >}} {{< tab "Use an SSH tunnel">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - CONNECTION`](/sql/create-connection/#ssh-tunnel) command to create an SSH - tunnel connection: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true" %}} - ```mzsql - CREATE CONNECTION ssh_connection TO SSH TUNNEL ( - HOST '', - PORT , - USER '' - ); - ``` + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" %}} - - Replace `` and ` with the public IP - address and port of the SSH bastion host you created [earlier](#b-optional-configure-network-security). +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-aurora-rds-self-hosted" indent="true" %}} - - Replace `` with the username for the key pair you - created for your SSH bastion host. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true" %}} -1. Get Materialize's public keys for the SSH tunnel connection you just - created: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} - ```mzsql - SELECT - mz_connections.name, - mz_ssh_tunnel_connections.* - FROM - mz_connections JOIN - mz_ssh_tunnel_connections USING(id) - WHERE - mz_connections.name = 'ssh_connection'; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-secret" indent="true" %}} -1. Log in to your SSH bastion host and add Materialize's public keys to the - `authorized_keys` file, for example: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true" %}} - ```sh - # Command for Linux - echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys - echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys - ``` + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-rds" indent="true" %}} -1. Back in the SQL client connected to Materialize, validate the SSH tunnel - connection you created using the [`VALIDATE CONNECTION`](/sql/validate-connection) command: - - ```mzsql - VALIDATE CONNECTION ssh_connection; - ``` - - If no validation error is returned, move to the next step. - -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the - password for the `materialize` PostgreSQL user you created - [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - SSH TUNNEL ssh_connection - ); - ``` - - - Replace `` with your RDS endpoint. To find your RDS endpoint, select - your database in the RDS Console, and look under **Connectivity & - security**. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +{{< /tab >}} -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your RDS instance and start ingesting data from the publication you - created [earlier](#2-create-a-publication-and-a-replication-user): +{{< /tabs >}} - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +### 3. Start ingesting data - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} -{{< /tab >}} +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} -{{< /tabs >}} +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/azure-db.md b/doc/user/content/ingest-data/postgres/azure-db.md index 09cdc71133d03..a4110d3d4450c 100644 --- a/doc/user/content/ingest-data/postgres/azure-db.md +++ b/doc/user/content/ingest-data/postgres/azure-db.md @@ -60,7 +60,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), +1. In the [Materialize console's SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -93,7 +93,7 @@ to serve as your SSH bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. 1. In the [Materialize console's SQL - Shell](https://console.materialize.com/), or your preferred SQL client + Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -113,7 +113,7 @@ to serve as your SSH bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -124,177 +124,80 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your PostgreSQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - SECRET`](/sql/create-secret/) command to securely store the password for the - `materialize` PostgreSQL user you created - [earlier](#2-create-a-publication-and-a-replication-user): +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a - connection object with access and authentication details for Materialize to - use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - SSL MODE 'require', - DATABASE '' - ); - ``` - - - Replace `` with your Azure DB endpoint. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. - -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize -to your Azure instance and start ingesting data from the publication you -created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` - - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true" %}} -1. After source creation, you can handle upstream [schema changes](/sql/create-source/postgres/#schema-changes) - for specific replicated tables using the [`ALTER SOURCE...ADD SUBSOURCE`](/sql/alter-source/#context) - and [`DROP SOURCE`](/sql/alter-source/#dropping-subsources) syntax. + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-general" indent="true" %}} {{< /tab >}} {{< tab "Use an SSH tunnel">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - CONNECTION`](/sql/create-connection/#ssh-tunnel) command to create an SSH - tunnel connection: - - ```mzsql - CREATE CONNECTION ssh_connection TO SSH TUNNEL ( - HOST '', - PORT , - USER '' - ); - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true" %}} - - Replace `` and ` with the public IP - address and port of the SSH bastion host you created [earlier](#b-optional-configure-network-security). + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" indent="true" %}} - - Replace `` with the username for the key pair you - created for your SSH bastion host. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-general" indent="true" %}} -1. Get Materialize's public keys for the SSH tunnel connection you just - created: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true" %}} - ```mzsql - SELECT - mz_connections.name, - mz_ssh_tunnel_connections.* - FROM - mz_connections JOIN - mz_ssh_tunnel_connections USING(id) - WHERE - mz_connections.name = 'ssh_connection'; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} -1. Log in to your SSH bastion host and add Materialize's public keys to the -`authorized_keys` file, for example: +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-secret" indent="true" %}} - ```sh - # Command for Linux - echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys - echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true" %}} -1. Back in the SQL client connected to Materialize, validate the SSH tunnel - connection you created using the [`VALIDATE CONNECTION`](/sql/validate-connection) - command: + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-general" indent="true" %}} - ```mzsql - VALIDATE CONNECTION ssh_connection; - ``` - - If no validation error is returned, move to the next step. - -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the - password for the `materialize` PostgreSQL user you created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - SSH TUNNEL ssh_connection - ); - ``` - - - Replace `` with your Azure DB endpoint. +{{< /tab >}} +{{< /tabs >}} - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +### 3. Start ingesting data -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize -to your Azure instance and start ingesting data from the publication you -created [earlier](#2-create-a-publication-and-a-replication-user): +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. - -{{< /tab >}} - -{{< /tabs >}} +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/cloud-sql.md b/doc/user/content/ingest-data/postgres/cloud-sql.md index 10ca1c8ffd0ef..f1168bd8ce8bd 100644 --- a/doc/user/content/ingest-data/postgres/cloud-sql.md +++ b/doc/user/content/ingest-data/postgres/cloud-sql.md @@ -61,7 +61,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), +1. In the [Materialize console's SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -94,7 +94,7 @@ network to allow traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. 1. In the [Materialize console's SQL - Shell](https://console.materialize.com/), or your preferred SQL client + Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -114,7 +114,7 @@ bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -125,30 +125,79 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your PostgreSQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} {{< tab "Allow Materialize IPs">}} -{{% postgres-direct/ingesting-data/allow-materialize-ips %}} +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-general" indent="true" %}} {{< /tab >}} {{< tab "Use an SSH tunnel">}} -{{% postgres-direct/ingesting-data/use-ssh-tunnel %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-general" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-secret" indent="true" %}} + +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true" %}} + + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-general" indent="true" %}} + {{< /tab >}} {{< /tabs >}} -### 3. Monitor the ingestion status +### 3. Start ingesting data + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} + +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/postgres/neon.md b/doc/user/content/ingest-data/postgres/neon.md index debc4757aef04..4f99adb726599 100644 --- a/doc/user/content/ingest-data/postgres/neon.md +++ b/doc/user/content/ingest-data/postgres/neon.md @@ -198,7 +198,7 @@ If you use Neon's [**IP Allow**](https://neon.tech/docs/introduction/ip-allow) feature to limit the IP addresses that can connect to your Neon instance, you will need to allow inbound traffic from Materialize IP addresses. -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), +1. In the [Materialize console's SQL Shell](/console/), or your preferred SQL client connected to Materialize, run the following query to find the static egress IP addresses, for the Materialize region you are running in: @@ -217,10 +217,10 @@ will need to allow inbound traffic from Materialize IP addresses. ## C. Ingest data in Materialize The steps in this section are specific to Materialize. You can run them in the -[Materialize console's SQL Shell](https://console.materialize.com/) or your +[Materialize console's SQL Shell](/console/) or your preferred SQL client connected to Materialize. -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL diff --git a/doc/user/content/ingest-data/postgres/self-hosted.md b/doc/user/content/ingest-data/postgres/self-hosted.md index 96e44eb52a15c..dd62cfddce760 100644 --- a/doc/user/content/ingest-data/postgres/self-hosted.md +++ b/doc/user/content/ingest-data/postgres/self-hosted.md @@ -84,7 +84,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), +1. In the [Materialize console's SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -213,7 +213,6 @@ option. be ready before you create a source. {{< /tab >}} - {{< tab "Use an SSH tunnel">}} To create an SSH tunnel from Materialize to your database, you launch an VM to @@ -233,7 +232,7 @@ traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. 1. In the [Materialize console's SQL - Shell](https://console.materialize.com/), or your preferred SQL client + Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -253,7 +252,7 @@ traffic from the bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your PostgreSQL @@ -264,225 +263,95 @@ scenarios, we recommend separating your workloads into multiple clusters for {{% postgres-direct/create-a-cluster %}} -### 2. Start ingesting data +### 2. Create a connection -Now that you've configured your database network and created an ingestion -cluster, you can connect Materialize to your PostgreSQL database and start -ingesting data. The exact steps depend on your networking configuration, so -start by selecting the relevant option. +Once you have configured your network, create a connection in Materialize per +your networking configuration. {{< tabs >}} {{< tab "Allow Materialize IPs">}} -1. In the SQL client connected to Materialize, use the [`CREATE - SECRET`](/sql/create-secret/) command to securely store the password for the - `materialize` PostgreSQL user you created - [earlier](#2-create-a-publication-and-a-replication-user): +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-secret" indent="true" %}} - ```mzsql - CREATE SECRET pgpass AS ''; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection" indent="true" %}} -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a - connection object with access and authentication details for Materialize to - use: + {{% include-example + file="examples/ingest_data/postgres/create_connection_ips_cloud" + example="create-connection-options-general" indent="true" %}} - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - SSL MODE 'require', - DATABASE '' - ); - ``` - - - Replace `` with your database endpoint. +{{< /tab >}} - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +{{< tab "Use AWS PrivateLink">}} -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your database and start ingesting data from the publication you created - [earlier](#2-create-a-publication-and-a-replication-user): +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-secret" indent="true" %}} - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection" indent="true" %}} - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. + {{% include-example + file="examples/ingest_data/postgres/create_connection_privatelink_cloud" + example="create-connection-options-self-hosted" indent="true" %}} {{< /tab >}} - {{< tab "Use an SSH tunnel">}} -1. In the [Materialize console's SQL Shell](https://console.materialize.com/), - or your preferred SQL client connected to Materialize, use the [`CREATE - CONNECTION`](/sql/create-connection/#ssh-tunnel) command to create an SSH - tunnel connection: - - ```mzsql - CREATE CONNECTION ssh_connection TO SSH TUNNEL ( - HOST '', - PORT , - USER '' - ); - ``` - - - Replace `` and ` with the public IP - address and port of the SSH bastion host you created [earlier](#b-optional-configure-network-security). - - - Replace `` with the username for the key pair you - created for your SSH bastion host. - -1. Get Materialize's public keys for the SSH tunnel connection you just - created: - - ```mzsql - SELECT - mz_connections.name, - mz_ssh_tunnel_connections.* - FROM - mz_connections JOIN - mz_ssh_tunnel_connections USING(id) - WHERE - mz_connections.name = 'ssh_connection'; - ``` - -1. Log in to your SSH bastion host and add Materialize's public keys to the - `authorized_keys` file, for example: - - ```sh - # Command for Linux - echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys - echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys - ``` - -1. Back in the SQL client connected to Materialize, validate the SSH tunnel - connection you created using the [`VALIDATE CONNECTION`](/sql/validate-connection) - command: - - ```mzsql - VALIDATE CONNECTION ssh_connection; - ``` - - If no validation error is returned, move to the next step. - -1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the - password for the `materialize` PostgreSQL user you created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER 'materialize', - PASSWORD SECRET pgpass, - DATABASE '', - SSH TUNNEL ssh_connection - ); - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection" indent="true" %}} - - Replace `` with your database endpoint. + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-ssh-tunnel-connection-options" indent="true" %}} - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="get-public-keys-aurora-rds-self-hosted" indent="true" %}} -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your Azure instance and start ingesting data from the publication you - created [earlier](#2-create-a-publication-and-a-replication-user): +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="login-to-ssh-bastion-host" indent="true" %}} - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="validate-ssh-tunnel-connection" indent="true" %}} - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-secret" indent="true" %}} -1. After source creation, you can handle upstream [schema changes](/sql/create-source/postgres/#schema-changes) - for specific replicated tables using the [`ALTER SOURCE...ADD SUBSOURCE`](/sql/alter-source/#context) - and [`DROP SOURCE`](/sql/alter-source/#dropping-subsources) syntax. +1. {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection" indent="true" %}} + {{% include-example + file="examples/ingest_data/postgres/create_connection_ssh_cloud" + example="create-connection-options-general" indent="true" %}} {{< /tab >}} -{{< tab "AWS PrivateLink">}} - -1. Back in the SQL client connected to Materialize, use the [`CREATE SECRET`](/sql/create-secret/) - command to securely store the password for the `materialize` PostgreSQL user you - created [earlier](#2-create-a-publication-and-a-replication-user): - - ```mzsql - CREATE SECRET pgpass AS ''; - ``` - -1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create - another connection object, this time with database access and authentication - details for Materialize to use: - - ```mzsql - CREATE CONNECTION pg_connection TO POSTGRES ( - HOST '', - PORT 5432, - USER postgres, - PASSWORD SECRET pgpass, - DATABASE , - AWS PRIVATELINK privatelink_svc - ); - ``` - - - Replace `` with your database endpoint. - - - Replace `` with the name of the database containing the tables - you want to replicate to Materialize. +{{< /tabs >}} -1. Use the [`CREATE SOURCE`](/sql/create-source/) command to connect Materialize - to your database and start ingesting data from the publication you created - [earlier](#2-create-a-publication-and-a-replication-user): +### 3. Start ingesting data - ```mzsql - CREATE SOURCE mz_source - IN CLUSTER ingest_postgres - FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') - FOR ALL TABLES; - ``` +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source" %}} - By default, the source will be created in the active cluster; to use a - different cluster, use the `IN CLUSTER` clause. To ingest data from - specific schemas or tables in your publication, use `FOR SCHEMAS - (,)` or `FOR TABLES (, )` instead of `FOR - ALL TABLES`. +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" example="create-source-options" %}} -{{< /tab >}} - -{{< /tabs >}} +{{% include-example file="examples/ingest_data/postgres/create_source_cloud" +example="schema-changes" %}} -### 3. Monitor the ingestion status +### 4. Monitor the ingestion status {{% postgres-direct/check-the-ingestion-status %}} -### 4. Right-size the cluster +### 5. Right-size the cluster {{% postgres-direct/right-size-the-cluster %}} diff --git a/doc/user/content/ingest-data/redpanda/redpanda-cloud.md b/doc/user/content/ingest-data/redpanda/redpanda-cloud.md index 99bb76e07de33..14f1fb73847b1 100644 --- a/doc/user/content/ingest-data/redpanda/redpanda-cloud.md +++ b/doc/user/content/ingest-data/redpanda/redpanda-cloud.md @@ -101,7 +101,7 @@ start by selecting the relevant option. 1. Copy the URL under **Cluster hosts**. This will be your `` going forward. -1. In the Materialize [SQL shell](https://console.materialize.com/), or your +1. In the Materialize [SQL shell](/console/), or your preferred SQL client, create a connection with your Redpanda Cloud cluster access and authentication details using the [`CREATE CONNECTION`](/sql/create-connection/) command: @@ -200,7 +200,7 @@ in a region supported by Materialize: `us-east-1`,`us-west-2`, or `eu-west-1`. '.cluster.aws_private_link' ``` -1. In the Materialize [SQL shell](https://console.materialize.com/), or your +1. In the Materialize [SQL shell](/console/), or your preferred SQL client, create a [PrivateLink connection](/ingest-data/network-security/privatelink/) using the service name from the previous step. Be sure to specify **all availability zones** of your Redpanda Cloud cluster. diff --git a/doc/user/content/ingest-data/sql-server/self-hosted.md b/doc/user/content/ingest-data/sql-server/self-hosted.md index 8d0691c9daa2d..98a44bcd44abe 100644 --- a/doc/user/content/ingest-data/sql-server/self-hosted.md +++ b/doc/user/content/ingest-data/sql-server/self-hosted.md @@ -57,7 +57,7 @@ Select the option that works best for you. {{< tab "Allow Materialize IPs">}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, find the static egress IP addresses for the Materialize region you are running in: @@ -205,7 +205,7 @@ traffic from the bastion host. 1. Configure the SSH bastion host to allow traffic only from Materialize. - 1. In the [SQL Shell](https://console.materialize.com/), or your preferred + 1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, get the static egress IP addresses for the Materialize region you are running in: @@ -225,7 +225,7 @@ traffic from the bastion host. ## C. Ingest data in Materialize -### 1. (Optional) Create a cluster +### 1. (Recommended) Create a cluster {{< note >}} If you are prototyping and already have a cluster to host your SQL Server diff --git a/doc/user/content/ingest-data/striim.md b/doc/user/content/ingest-data/striim.md index fd731b7999e28..cfb2b12d36b17 100644 --- a/doc/user/content/ingest-data/striim.md +++ b/doc/user/content/ingest-data/striim.md @@ -90,7 +90,7 @@ scenarios, we recommend separating your workloads into multiple clusters for [resource isolation](/sql/create-cluster/#resource-isolation). {{< /note >}} -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, use the [`CREATE CONNECTION`](/sql/create-connection/) command to create connection objects with access and authentication details to your Kafka cluster and schema registry: diff --git a/doc/user/content/ingest-data/troubleshooting.md b/doc/user/content/ingest-data/troubleshooting.md index 641108583a705..7cdbe40108f35 100644 --- a/doc/user/content/ingest-data/troubleshooting.md +++ b/doc/user/content/ingest-data/troubleshooting.md @@ -28,7 +28,7 @@ troubleshooting](/transform-data/troubleshooting) guide instead. ## Why isn't my source ingesting data? First, check the status of your source in the Materialize console by navigating -to https://console.materialize.com/, clicking the **Sources** tab in the +to /console/, clicking the **Sources** tab in the navigation bar, and clicking the affected source. Alternatively, you can get this information from the system catalog by querying diff --git a/doc/user/content/ingest-data/webhooks/amazon-eventbridge.md b/doc/user/content/ingest-data/webhooks/amazon-eventbridge.md index bd3e4030791a3..10dd41fbee810 100644 --- a/doc/user/content/ingest-data/webhooks/amazon-eventbridge.md +++ b/doc/user/content/ingest-data/webhooks/amazon-eventbridge.md @@ -121,7 +121,7 @@ validation. With the source set up in Materialize and the API destination configured in Amazon EventBridge, you can now query the incoming data: -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/hubspot.md b/doc/user/content/ingest-data/webhooks/hubspot.md index 2ca1f4fc1f831..8c6b7828c7bda 100644 --- a/doc/user/content/ingest-data/webhooks/hubspot.md +++ b/doc/user/content/ingest-data/webhooks/hubspot.md @@ -160,7 +160,7 @@ Materialize. With the source set up in Materialize and the webhook workflow configured in HubSpot, you can now query the incoming data: -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/rudderstack.md b/doc/user/content/ingest-data/webhooks/rudderstack.md index 47fa4abe0a0be..f739d9b7ffdda 100644 --- a/doc/user/content/ingest-data/webhooks/rudderstack.md +++ b/doc/user/content/ingest-data/webhooks/rudderstack.md @@ -134,7 +134,7 @@ On the **Connection Settings** page: With the source set up in Materialize and the webhook destination configured in Rudderstack, you can now query the incoming data: -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/segment.md b/doc/user/content/ingest-data/webhooks/segment.md index 99c8d30c6a14a..3a96e4ae247c5 100644 --- a/doc/user/content/ingest-data/webhooks/segment.md +++ b/doc/user/content/ingest-data/webhooks/segment.md @@ -155,7 +155,7 @@ mapping: With the source set up in Materialize and the webhook destination configured in Segment, you can now query the incoming data: -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/snowcatcloud.md b/doc/user/content/ingest-data/webhooks/snowcatcloud.md index 21d45b2ea8acb..ea3f6c46698b7 100644 --- a/doc/user/content/ingest-data/webhooks/snowcatcloud.md +++ b/doc/user/content/ingest-data/webhooks/snowcatcloud.md @@ -131,7 +131,7 @@ On the **Materialize Settings** page: With the source set up in Materialize and the webhook destination configured in SnowcatCloud, you can now query the incoming data: -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/stripe.md b/doc/user/content/ingest-data/webhooks/stripe.md index 7f2e70e5ad05f..228d90defbb92 100644 --- a/doc/user/content/ingest-data/webhooks/stripe.md +++ b/doc/user/content/ingest-data/webhooks/stripe.md @@ -132,7 +132,7 @@ Stripe signing scheme, check out the [Stripe documentation](https://stripe.com/d ## Step 5. Validate incoming data -1. [In the Materialize console](https://console.materialize.com/), navigate to +1. [In the Materialize console](/console/), navigate to the **SQL Shell**. 1. Use SQL queries to inspect and analyze the incoming data: diff --git a/doc/user/content/ingest-data/webhooks/webhook-quickstart.md b/doc/user/content/ingest-data/webhooks/webhook-quickstart.md index cd369cd77c589..bb151ca29ff06 100644 --- a/doc/user/content/ingest-data/webhooks/webhook-quickstart.md +++ b/doc/user/content/ingest-data/webhooks/webhook-quickstart.md @@ -24,7 +24,7 @@ you to learn and prototype with no external dependencies. All you need is a Materialize account. If you already have one — great! If not, [sign up for a free trial account](https://materialize.com/register/?utm_campaign=General&utm_source=documentation) first. -When you're ready, head over to the [Materialize console](https://console.materialize.com/), +When you're ready, head over to the [Materialize console](/console/), and pop open the SQL Shell. ## Step 1. Create a secret diff --git a/doc/user/content/integrations/_index.md b/doc/user/content/integrations/_index.md index 0687567bdcd87..d2f87c7f3527d 100644 --- a/doc/user/content/integrations/_index.md +++ b/doc/user/content/integrations/_index.md @@ -13,7 +13,7 @@ menu: main: identifier: "integrations" name: "Tools and integrations" - weight: 25 + weight: 70 --- Materialize is **wire-compatible** with PostgreSQL and can integrate with many diff --git a/doc/user/content/integrations/sql-clients.md b/doc/user/content/integrations/sql-clients.md index b4176291973dd..488fb5636b264 100644 --- a/doc/user/content/integrations/sql-clients.md +++ b/doc/user/content/integrations/sql-clients.md @@ -19,7 +19,7 @@ connect to your Materialize region using some common SQL clients. ## Connection parameters You can find the credentials for your Materialize region in the -[Materialize console](https://console.materialize.com/), under **Connect +[Materialize console](/console/), under **Connect externally** in the navigation bar. Field | Value diff --git a/doc/user/content/license.md b/doc/user/content/license.md index 145de1f88a049..506fefe05b0d1 100644 --- a/doc/user/content/license.md +++ b/doc/user/content/license.md @@ -7,5 +7,43 @@ menu: weight: 15 --- -Materialize is licensed under the Business Source License agreement [available -in the public GitHub repo here](https://github.com/MaterializeInc/materialize/blob/main/LICENSE). +## Materialize Cloud + +Materialize Cloud is licensed under the Business Source License agreement +[available in the public GitHub repo +here](https://github.com/MaterializeInc/materialize/blob/main/LICENSE). + +## Materialize Self-Managed + +Use of the Materialize Self-Managed Community edition is subject to the +[Materialize Business Source +License](https://github.com/MaterializeInc/materialize/blob/main/LICENSE) +agreement. + +To purchase a Materialize Self-Managed Enterprise License, contact us +[here](https://materialize.com/self-managed). + +### Self-Managed Community Edition Privacy Policy + +By downloading Materialize Self-Managed Community Edition, you are agreeing to +Materialize' [privacy policy](https://materialize.com/privacy-policy/). + +### Materialize Self-Managed Community Edition and Emulator Privacy FAQ + +When you use the Materialize Self-Managed Community Edition or the Materialize Emulator, we may collect the following information from the machine that runs the Materialize Self-Managed Community Edition or the Materialize Emulator software: + +- The IP address of the machine running Materialize. + +- If available, the cloud provider and region of the machine running + Materialize. + +- Usage data about your use of the product such as the types or quantity of + commands you run, the number of clusters or indexes you are running, and + similar feature usage information. + +The collection of this data is subject to the [Materialize Privacy Policy](https://materialize.com/privacy-policy/). + +Please note that if you visit our website or otherwise engage with us outside of +downloading the Materialize Self-Managed Community Edition or the Materialize +Emulator, we may collect additional information about you as further described +in our [Privacy Policy](https://materialize.com/privacy-policy/). diff --git a/doc/user/content/manage/_index.md b/doc/user/content/manage/_index.md index dd282303b715c..2034ced27a231 100644 --- a/doc/user/content/manage/_index.md +++ b/doc/user/content/manage/_index.md @@ -5,7 +5,7 @@ disable_list: true menu: main: identifier: "manage" - weight: 14 + weight: 50 --- This section contains various resources for managing Materialize. @@ -18,14 +18,6 @@ This section contains various resources for managing Materialize. | [Monitoring and alerting](/manage/monitor/) | Guides to set up monitoring and alerting | | [Disaster Recovery](/manage/disaster-recovery/) | Disaster recovery strategies | -## Access management guides - -| Guide | Description | -|-------|-------------| -| [User and service accounts](/manage/users-service-accounts/) | Manage user/service accounts | -| [Access control](/manage/access-control/) | Reference for role-based access management (RBAC) | -| [Manage network policies](/manage/manage-network-policies/) | Set up network policies | - ## Manage via dbt/Terraform | Guide | Description | diff --git a/doc/user/content/manage/dbt/get-started.md b/doc/user/content/manage/dbt/get-started.md index 34df123f7b7ff..b55c13107cec6 100644 --- a/doc/user/content/manage/dbt/get-started.md +++ b/doc/user/content/manage/dbt/get-started.md @@ -778,7 +778,7 @@ SELECT NULL AS col_with_constraints, the compiled statements under `/target/run` and `target/compiled` in the dbt project folder. -1. Using the [SQL Shell](https://console.materialize.com/), or your preferred +1. Using the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, double-check that all objects have been created: @@ -893,7 +893,7 @@ trigger **real-time alerts** downstream. This guarantees that your tests keep running in the background as views that are automatically updated as soon as an assertion fails. -1. Using the [SQL Shell](https://console.materialize.com/), or your preferred +1. Using the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, that the schema storing the tests has been created, as well as the test materialized views: diff --git a/doc/user/content/manage/disaster-recovery/_index.md b/doc/user/content/manage/disaster-recovery/_index.md index 0091ca3cafd41..a0aab16fd768c 100644 --- a/doc/user/content/manage/disaster-recovery/_index.md +++ b/doc/user/content/manage/disaster-recovery/_index.md @@ -1,6 +1,6 @@ --- -title: "Disaster recovery" -description: "Learn about various disaster recovery (DR) strategies for Materialize." +title: "Disaster recovery (Cloud)" +description: "Learn about various disaster recovery (DR) strategies for Materialize Cloud." disable_list: true menu: main: diff --git a/doc/user/content/manage/disaster-recovery/recovery-characteristics.md b/doc/user/content/manage/disaster-recovery/recovery-characteristics.md index da86726d50dfc..bfd83a3bb1353 100644 --- a/doc/user/content/manage/disaster-recovery/recovery-characteristics.md +++ b/doc/user/content/manage/disaster-recovery/recovery-characteristics.md @@ -1,6 +1,6 @@ --- -title: "Materialize DR characteristics" -description: "Reference on various failure modes and recovery characteristics of Materialize." +title: "Materialize Cloud DR characteristics" +description: "Reference on various failure modes and recovery characteristics of Materializ Cloud." menu: main: parent: "disaster-recovery" diff --git a/doc/user/content/manage/monitor/alerting.md b/doc/user/content/manage/monitor/alerting.md index 65af2e11874b3..e3cf4efada212 100644 --- a/doc/user/content/manage/monitor/alerting.md +++ b/doc/user/content/manage/monitor/alerting.md @@ -44,7 +44,7 @@ Credits | Consumption rate increase by X% | Consumption rate increase by Y% | Av ## Maintenance window -Materialize has a [release and a maintenance window almost every week](/releases/) at a defined [schedule](/releases/#schedule). +Materialize Cloud has a [release and a maintenance window almost every week](/releases/) at a defined [schedule](/releases/#release-schedules). During an upgrade, clients may experience brief connection interruptions, but the service otherwise remains fully available. Alerts may get triggered during this brief period of time. For this case, you can configure your monitoring tool to avoid unnecessary alerts as follows: diff --git a/doc/user/content/manage/monitor/datadog.md b/doc/user/content/manage/monitor/datadog.md index 0d44922d97c9c..ed40bf3407b48 100644 --- a/doc/user/content/manage/monitor/datadog.md +++ b/doc/user/content/manage/monitor/datadog.md @@ -45,7 +45,7 @@ which has been tried and tested in production environments. 1. In the configuration file, define the connection to your Materialize region - under `connections` using the credentials provided in the [Materialize Console](https://console.materialize.com/). + under `connections` using the credentials provided in the [Materialize Console](/console/). {{< note >}} You must escape the special `@` character in `USER` for a successful diff --git a/doc/user/content/manage/monitor/grafana.md b/doc/user/content/manage/monitor/grafana.md index 859481eb09dfd..0970e2cf631aa 100644 --- a/doc/user/content/manage/monitor/grafana.md +++ b/doc/user/content/manage/monitor/grafana.md @@ -44,7 +44,7 @@ which has been tried and tested in production environments. {{}} 2. In the configuration file, define the connection to your Materialize region - under `connections` using the credentials provided in the [Materialize Console](https://console.materialize.com/). + under `connections` using the credentials provided in the [Materialize Console](/console/). {{< note >}} You must escape the special `@` character in `USER` for a successful diff --git a/doc/user/content/manage/operational-guidelines.md b/doc/user/content/manage/operational-guidelines.md index 374b3272f156b..07611e95672d7 100644 --- a/doc/user/content/manage/operational-guidelines.md +++ b/doc/user/content/manage/operational-guidelines.md @@ -71,6 +71,4 @@ See also [Cluster architecture](#three-tier-architecture). ## Role-based access control (RBAC) -{{< include-md file="shared-content/rbac/rbac-intro-disambiguation.md" >}} - {{% yaml-sections data="rbac/recommendations" heading-field="recommendation" heading-level=3 %}} diff --git a/doc/user/content/manage/terraform/appendix-secret-stores.md b/doc/user/content/manage/terraform/appendix-secret-stores.md index d7a67652fbb64..77c57e3840705 100644 --- a/doc/user/content/manage/terraform/appendix-secret-stores.md +++ b/doc/user/content/manage/terraform/appendix-secret-stores.md @@ -5,7 +5,7 @@ disable_toc: true menu: main: parent: "manage-terraform" - weight: 25 + weight: 50 --- Materialize does not directly integrate with external secret stores, but it's possible to manage this integration via Terraform. diff --git a/doc/user/content/manage/access-control/rbac-terraform-tutorial.md b/doc/user/content/manage/terraform/manage-rbac.md similarity index 98% rename from doc/user/content/manage/access-control/rbac-terraform-tutorial.md rename to doc/user/content/manage/terraform/manage-rbac.md index 7ba949b0f15d6..13e3243e63bf4 100644 --- a/doc/user/content/manage/access-control/rbac-terraform-tutorial.md +++ b/doc/user/content/manage/terraform/manage-rbac.md @@ -1,10 +1,12 @@ --- -title: "Tutorial: Manage privileges with Terraform" +title: "Manage privileges with Terraform" description: "Add users, create roles, and assign privileges in Materialize with Terraform" menu: main: - parent: access-control + parent: manage-terraform weight: 30 +aliases: + - /manage/access-control/rbac-terraform-tutorial/ --- This tutorial walks you through managing roles in Materialize with [Terraform](https://www.terraform.io/). By the end of this tutorial you will: diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index 5e64a705a9e90..934d433226df8 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -1,20 +1,27 @@ --- -title: "Release schedule" -description: "How Materialize is released" +title: "Release Notes" +description: "Section containining release info." disable_list: true menu: main: - parent: releases-previews - weight: 5 - identifier: releases + identifier: "releases" + weight: 80 --- -We are continually improving Materialize with new features and bug fixes. We -periodically release these improvements to your Materialize account. This page -describes the changes in each release and the process by which they are -deployed. +## Releases -## Schedule + +### Self-Managed release v25.3.0 + +#### License keys + +#### Swap support + +#### SQL Server + + + +## Materialize Cloud Upgrade Schedule Materialize upgrades all regions to the latest release each week according to the following schedule: diff --git a/doc/user/content/security/_index.md b/doc/user/content/security/_index.md new file mode 100644 index 0000000000000..38af35a0fe223 --- /dev/null +++ b/doc/user/content/security/_index.md @@ -0,0 +1,17 @@ +--- +title: "Security" +description: "" +disable_list: true +menu: + main: + identifier: "security" + weight: 45 +--- + +## Access management guides + +| Guide | Description | +|-------|-------------| +| [User and service accounts](/manage/users-service-accounts/) | Manage user/service accounts | +| [Access control](/manage/access-control/) | Reference for role-based access management (RBAC) | +| [Manage network policies](/manage/manage-network-policies/) | Set up network policies | diff --git a/doc/user/content/manage/access-control/appendix-built-in-roles.md b/doc/user/content/security/appendix-built-in-roles.md similarity index 98% rename from doc/user/content/manage/access-control/appendix-built-in-roles.md rename to doc/user/content/security/appendix-built-in-roles.md index 32b222b4dadc1..47b1c33e5ce3b 100644 --- a/doc/user/content/manage/access-control/appendix-built-in-roles.md +++ b/doc/user/content/security/appendix-built-in-roles.md @@ -3,7 +3,7 @@ title: "Appendix: Built-in roles" description: "List of predefined built-in roles in Materialize." menu: main: - parent: access-control + parent: security weight: 100 --- diff --git a/doc/user/content/manage/access-control/appendix-command-privileges.md b/doc/user/content/security/appendix-command-privileges.md similarity index 85% rename from doc/user/content/manage/access-control/appendix-command-privileges.md rename to doc/user/content/security/appendix-command-privileges.md index b2c62001b87e7..b3b17731e8e0e 100644 --- a/doc/user/content/manage/access-control/appendix-command-privileges.md +++ b/doc/user/content/security/appendix-command-privileges.md @@ -3,7 +3,7 @@ title: "Appendix: Privileges by commands" description: "" menu: main: - parent: access-control + parent: security weight: 99 disable_toc: true --- diff --git a/doc/user/content/manage/access-control/appendix-privileges.md b/doc/user/content/security/appendix-privileges.md similarity index 94% rename from doc/user/content/manage/access-control/appendix-privileges.md rename to doc/user/content/security/appendix-privileges.md index 059a8d6f0bba3..dcb728d92d96d 100644 --- a/doc/user/content/manage/access-control/appendix-privileges.md +++ b/doc/user/content/security/appendix-privileges.md @@ -3,7 +3,7 @@ title: "Appendix: Privileges" description: "List of available privileges in Materialize." menu: main: - parent: access-control + parent: security weight: 95 --- diff --git a/doc/user/content/security/cloud/_index.md b/doc/user/content/security/cloud/_index.md new file mode 100644 index 0000000000000..1b0df8f2ec570 --- /dev/null +++ b/doc/user/content/security/cloud/_index.md @@ -0,0 +1,136 @@ +--- +title: "Materialize Cloud" +description: "Manage users and service accounts for Materialize Cloud." +disable_list: true +menu: + main: + parent: "security" + identifier: "security-cloud" + weight: 10 +--- + +As an administrator of a Materialize organization, you can manage the users and +apps (via service accounts) that can access your Materialize organization and +resources. + +## Organization roles + +During creation of a user/service account in Materialize, the account is +assigned an organization role: + +{{< include-md file="shared-content/rbac/organization-roles.md" >}} + +## User accounts + +As an **Organization admin**, you can [invite new +users](./invite-users/) via the Materialize Console. When you invite a new user, +Materialize will email the user with an invitation link. + +{{< include-md file="shared-content/rbac/invite-user-note.md" >}} + +For instructions on inviting users to your Materialize organization, see [Invite +users](./invite-users/). + +## Service accounts + +{{< tip >}} + +As a best practice, we recommend you use service accounts to connect external +applications and services to Materialize. + +{{}} + +As an **Organization admin**, you can create a new service account via +the [Materialize Console](/console/) or via +[Terraform](/manage/terraform/). + +{{< note >}} + +- The new account creation is not finished until the first time you connect with +the account. + +- {{< include-md file="shared-content/rbac/service-account-creation.md" >}} + +{{}} + +For instructions on creating a new service account in your Materialize +organization, see [Create service accounts](./create-service-accounts/). + +## Single sign-on (SSO) + +As an **Organization admin**, you can configure single sign-on (SSO) as +an additional layer of account security using your existing +[SAML](https://auth0.com/blog/how-saml-authentication-works/)- or [OpenID +Connect](https://auth0.com/intro-to-iam/what-is-openid-connect-oidc)-based +identity provider. This ensures that all users can securely log in to the +Materialize Console using the same authentication scheme and credentials across +all systems in your organization. + +To configure SSO for your Materialize organization, follow [this step-by-step +guide](./sso/). + +{{< annotation type="Disambiguation" >}} +{{< include-md file="shared-content/rbac/rbac-intro-disambiguation.md" >}} + +This section focuses on the database access control. For information on +organization roles, see [Users and service +accounts](/manage/users-service-accounts/). +{{}} + + + +## Role-based access control + +In Materialize, role-based access control (RBAC) governs access to **database +objects** through privileges granted to [database +roles](/manage/access-control/manage-roles/). + +## Roles and privileges + +{{% include-md file="shared-content/rbac/db-roles.md" %}} + +### Managing privileges + +{{% include-md file="shared-content/rbac/db-roles-managing-privileges.md" %}} + +{{< annotation type="Disambiguation" >}} +{{% include-md file="shared-content/rbac/grant-vs-alter-default-privilege.md" +%}} +{{}} + +### Initial privileges + +{{< include-md file="shared-content/rbac/db-roles-initial-privileges.md" >}} + +You can modify the privileges of your organization's `PUBLIC` role as well as +the modify default privileges for `PUBLIC`. + +## Privilege inheritance and modular access control + +In Materialize, when you grant a role to another role (user role/service account +role/independent role), the target role inherits privileges through the granted +role. + +In general, to grant a user or service account privileges, create roles with the +desired privileges and grant these roles to the database role associated with +the user/service account email/name. Although you can grant privileges directly +to the associated roles, using separate, reusable roles is recommended for +better access management. + +With privilege inheritance, you can compose more complex roles by +combining existing roles, enabling modular access control. However: + +- Inheritance only applies to role privileges; role attributes and parameters + are not inherited. +- {{% include-md file="shared-content/rbac/revoke-roles-consideration.md" %}} + +## Best practices + +{{% yaml-sections data="rbac/recommendations" heading-field="recommendation" heading-level=3 %}} + + +## See also + +- [Role-based access control](/manage/access-control/) +- [Manage with dbt](/manage/dbt/) +- [Manage with Terraform](/manage/terraform/) diff --git a/doc/user/content/manage/manage-network-policies.md b/doc/user/content/security/cloud/manage-network-policies.md similarity index 95% rename from doc/user/content/manage/manage-network-policies.md rename to doc/user/content/security/cloud/manage-network-policies.md index ba0dac74099df..64e69aa516d60 100644 --- a/doc/user/content/manage/manage-network-policies.md +++ b/doc/user/content/security/cloud/manage-network-policies.md @@ -1,14 +1,14 @@ --- -title: "Manage network policies" +title: "Manage network policies (Cloud)" description: "Manage/configure network policies to restrict access to a Materialize region using IP-based rules." aliases: - /manage/access-control/manage-network-policies/ menu: main: - parent: "manage" - weight: 14 + parent: "security-cloud" + weight: 30 name: "Manage network policies" - identifier: "manage-network-policies" + identifier: "security-network-policies" --- {{< tip >}} diff --git a/doc/user/content/manage/access-control/manage-roles.md b/doc/user/content/security/cloud/manage-roles.md similarity index 99% rename from doc/user/content/manage/access-control/manage-roles.md rename to doc/user/content/security/cloud/manage-roles.md index 5e39c69b8b554..62bcd09653669 100644 --- a/doc/user/content/manage/access-control/manage-roles.md +++ b/doc/user/content/security/cloud/manage-roles.md @@ -1,9 +1,9 @@ --- title: "Manage database roles" -description: "Create and manage database roles and privileges in Materialize" +description: "Create and manage database roles and privileges in Cloud Materialize." menu: main: - parent: access-control + parent: security-cloud weight: 15 aliases: - /sql/builtin-roles/ diff --git a/doc/user/content/manage/users-service-accounts/_index.md b/doc/user/content/security/cloud/users-service-accounts/_index.md similarity index 96% rename from doc/user/content/manage/users-service-accounts/_index.md rename to doc/user/content/security/cloud/users-service-accounts/_index.md index a8d9362f3a868..c99a178ad511c 100644 --- a/doc/user/content/manage/users-service-accounts/_index.md +++ b/doc/user/content/security/cloud/users-service-accounts/_index.md @@ -4,7 +4,7 @@ description: "Manage users and service accounts." disable_list: true menu: main: - parent: "manage" + parent: "security-cloud" identifier: "user-service-accounts" weight: 10 --- @@ -41,7 +41,7 @@ applications and services to Materialize. {{}} As an **Organization admin**, you can create a new service account via -the [Materialize Console](https://console.materialize.com/) or via +the [Materialize Console](/console/) or via [Terraform](/manage/terraform/). {{< note >}} diff --git a/doc/user/content/manage/users-service-accounts/create-service-accounts.md b/doc/user/content/security/cloud/users-service-accounts/create-service-accounts.md similarity index 98% rename from doc/user/content/manage/users-service-accounts/create-service-accounts.md rename to doc/user/content/security/cloud/users-service-accounts/create-service-accounts.md index 7ee578848a9de..fe89b7c99c965 100644 --- a/doc/user/content/manage/users-service-accounts/create-service-accounts.md +++ b/doc/user/content/security/cloud/users-service-accounts/create-service-accounts.md @@ -29,7 +29,7 @@ the account. ## Materialize Console -1. [Log in to the Materialize Console](https://console.materialize.com/). +1. [Log in to the Materialize Console](/console/). 1. In the side navigation bar, click **+ Create New** > **App Password**. diff --git a/doc/user/content/manage/users-service-accounts/invite-users.md b/doc/user/content/security/cloud/users-service-accounts/invite-users.md similarity index 92% rename from doc/user/content/manage/users-service-accounts/invite-users.md rename to doc/user/content/security/cloud/users-service-accounts/invite-users.md index fdd86509a657e..9191bbab0d011 100644 --- a/doc/user/content/manage/users-service-accounts/invite-users.md +++ b/doc/user/content/security/cloud/users-service-accounts/invite-users.md @@ -16,7 +16,7 @@ menu: As an **Organization administrator**, you can invite new users via the Materialize Console. -1. [Log in to the Materialize Console](https://console.materialize.com/). +1. [Log in to the Materialize Console](/console/). 1. Navigate to **Account** > **Account Settings** > **Users**. diff --git a/doc/user/content/manage/users-service-accounts/sso.md b/doc/user/content/security/cloud/users-service-accounts/sso.md similarity index 96% rename from doc/user/content/manage/users-service-accounts/sso.md rename to doc/user/content/security/cloud/users-service-accounts/sso.md index ac6d21eee6d7d..8af7302c549ed 100644 --- a/doc/user/content/manage/users-service-accounts/sso.md +++ b/doc/user/content/security/cloud/users-service-accounts/sso.md @@ -33,7 +33,7 @@ To make Materialize metadata available to Datadog, you must configure and run th ## Configure authentication -* [Log in to the Materialize console](https://console.materialize.com/). +* [Log in to the Materialize console](/console/). * Navigate to **Account** > **Account Settings** > **SSO**. diff --git a/doc/user/content/manage/access-control/_index.md b/doc/user/content/security/self-managed/_index.md similarity index 63% rename from doc/user/content/manage/access-control/_index.md rename to doc/user/content/security/self-managed/_index.md index aae6e1230931e..1274624ffcd66 100644 --- a/doc/user/content/manage/access-control/_index.md +++ b/doc/user/content/security/self-managed/_index.md @@ -1,36 +1,45 @@ --- -title: "Access control (Role-based)" -description: "How to configure and manage role-based database access control (RBAC) in Materialize." +title: "Materialize Self-Managed" +description: "Manage authentication and access control for Materialize Self-Managed." disable_list: true -aliases: - - /manage/access-control/rbac/ menu: main: - parent: manage - identifier: 'access-control' - weight: 12 + parent: "security" + identifier: "security-sm" + weight: 20 --- -{{< annotation type="Disambiguation" >}} -{{< include-md file="shared-content/rbac/rbac-intro-disambiguation.md" >}} +## Enabling authentication -This section focuses on the database access control. For information on -organization roles, see [Users and service -accounts](/manage/users-service-accounts/). -{{}} - +## Creating users + ## Role-based access control -In Materialize, role-based access control (RBAC) governs access to **database -objects** through privileges granted to [database +In Materialize, role-based access control (RBAC) governs access to objects +through privileges granted to [database roles](/manage/access-control/manage-roles/). +{{< note >}} +Initially, only the `mz_system` user (which has superuser/administrator +privileges) is available to manage roles. +{{}} + + + +## Enabling RBAC + +{{< include-md file="shared-content/rbac/enable-rbac.md" >}} + ## Roles and privileges {{% include-md file="shared-content/rbac/db-roles.md" %}} +- {{< include-md file="shared-content/rbac/create-users.md" >}} + +- {{< include-md file="shared-content/rbac/create-functional-roles.md" >}} + ### Managing privileges {{% include-md file="shared-content/rbac/db-roles-managing-privileges.md" %}} @@ -54,10 +63,9 @@ role/independent role), the target role inherits privileges through the granted role. In general, to grant a user or service account privileges, create roles with the -desired privileges and grant these roles to the database role associated with -the user/service account email/name. Although you can grant privileges directly -to the associated roles, using separate, reusable roles is recommended for -better access management. +desired privileges and grant these roles to the user or service account role. +Although you can grant privileges directly to the user or service account role, +using separate, reusable roles is recommended for better access management. With privilege inheritance, you can compose more complex roles by combining existing roles, enabling modular access control. However: diff --git a/doc/user/content/security/self-managed/configure-authentication.md b/doc/user/content/security/self-managed/configure-authentication.md new file mode 100644 index 0000000000000..c0b568144e90f --- /dev/null +++ b/doc/user/content/security/self-managed/configure-authentication.md @@ -0,0 +1,88 @@ +--- +title: "Configure authentication" +description: "Configure authentication in Materialize Self-Managed," +menu: + main: + parent: security-sm + identifier: configure-authentication + weight: 10 +--- + +## Configuring Authentication Type + +To configure the authentication type used by self-managed Materialize, use the +`spec.authenticatorKind` setting in conjunction with any specific configuration +for the authentication method. + +The `spec.authenticatorKind` setting determines which authentication method is +used: + +{{% yaml-table data="self_managed/authentication_setting" %}} + +## Configuring password authentication + +{{< public-preview >}}This feature{{}} + +Password authentication requires users to log in with a password. + +To configure self-managed Materialize for password authentication: + + Configuration | Description +---------------| ------------ +`spec.authenticatorKind` | Set to `Password` to enable password authentication. +`external_login_password_mz_system` | To the Kubernetes Secret referenced by `spec.backendSecretName`, add the secret key `external_login_password_mz_system`. This is the password for the `mz_system` user [^1], who is the only user initially available when password authentication is enabled. + +For example, if using Kind, in the `sample-materialize.yaml` file: + +```hc {hl_lines="14 24"} +apiVersion: v1 +kind: Namespace +metadata: + name: materialize-environment +--- +apiVersion: v1 +kind: Secret +metadata: + name: materialize-backend + namespace: materialize-environment +stringData: + metadata_backend_url: "..." + persist_backend_url: "..." + external_login_password_mz_system: "enter_mz_system_password" +--- +apiVersion: materialize.cloud/v1alpha1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v0.147.2 + backendSecretName: materialize-backend + authenticatorKind: Password +``` + +#### Logging in and creating users + +Initially, only the `mz_system` user [^1] is available. To create additional +users: + +1. Login as the `mz_system` user, using the +`external_login_password_mz_system` password, +![Image of Materialize Console login screen with mz_system user](/images/mz_system_login.png +"Materialize Console login screen with mz_system user") + +1. Use [`CREATE ROLE ... WITH LOGIN PASSWORD ...`](/sql/create-role) to create + new users: + + ```mzsql + CREATE ROLE WITH LOGIN PASSWORD ''; + ``` + +[^1]: The `mz_system` user is also used by the Materialize Operator for upgrades +and maintenance tasks. + +## Enabling RBAC + +{{< include-md file="shared-content/enable-rbac.md" >}} + +See [Access Control](/manage/access-control/) for details on role based authorization. diff --git a/doc/user/content/security/self-managed/manage-roles.md b/doc/user/content/security/self-managed/manage-roles.md new file mode 100644 index 0000000000000..fe14e8bc46f52 --- /dev/null +++ b/doc/user/content/security/self-managed/manage-roles.md @@ -0,0 +1,678 @@ +--- +title: "Manage database role" +description: "Create and manage database roles and privileges in Self-Managed Materialize." +menu: + main: + parent: security-sm + weight: 25 +--- + +In Materialize, role-based access control (RBAC) governs access to objects +through privileges granted to database roles. + +## Enabling RBAC + +{{< include-md file="shared-content/rbac/enable-rbac.md" >}} + +## Required privileges for managing roles + +{{< note >}} +Initially, only the `mz_system` user (which has superuser/administrator +privileges) is available to manage roles. +{{}} + +| Role management operations | Required privileges | +| ----------------------------------- | ------------------------ | +| To create/revoke/grant roles | {{< include-md +file="shared-content/sql-command-privileges/create-role.md" >}} {{< warning >}} +{{< include-md file="shared-content/rbac/createrole-consideration.md" >}} +{{}} | +| To view privileges for a role | None | +| To grant/revoke role privileges | {{< include-md +file="shared-content/sql-command-privileges/grant-privilege.md" >}}| +| To alter default privileges | {{< include-md +file="shared-content/sql-command-privileges/alter-default-privileges.md" >}} | + +See also [Appendix: Privileges by +command](/manage/access-control/appendix-command-privileges/) + +## Create a role + +{{< include-md file="shared-content/rbac/db-roles.md" >}} + +### Create individual user/service account roles + +{{< include-md file="shared-content/rbac/create-users.md" >}} + +{{< annotation type="Privilege(s) required to run the command" >}} +{{< include-md file="shared-content/sql-command-privileges/create-role.md" >}} +{{}} + +For example, the following creates: + +- A new user `blue.berry@example.com` (or more specifically, a new user role). +- A new service account `sales_report_app` (or more specifically, a new service + account role). + +{{< tabs >}} +{{< tab "A new user role" >}} +{{< include-example file="examples/rbac/create_roles" +example="create-role-user" >}} +{{}} +{{< tab "A new service account role" >}} +{{< include-example file="examples/rbac/create_roles" +example="create-role-service-account" >}} +{{}} +{{}} + +In Materialize, a role is created with inheritance support. With inheritance, +when a role is granted to another role (i.e., the target role), the target role +inherits privileges (not role attributes and parameters) through the other role. +{{< include-md file="shared-content/rbac/db-roles-public-membership.md" >}} + +{{% include-md file="shared-content/rbac/db-roles-managing-privileges.md" %}} + +{{< annotation type="Disambiguation" >}} +{{% include-md file="shared-content/rbac/grant-vs-alter-default-privilege.md" +%}} +{{}} + +See also: + +- For a list of required privileges for specific operations, see [Appendix: +Privileges by command](/manage/access-control/appendix-command-privileges/). + +### Create functional roles + +{{< include-md file="shared-content/rbac/create-functional-roles.md" >}} + +{{< tip >}} +{{< include-md file="shared-content/rbac/role-name-restrictions.md" >}} +{{}} + +For example, the following creates: +- A role for users who need to perform compute/transform operations in the + compute/transform. +- A role for users who need to manage indexes on the serving cluster(s). +- A role for users who need to read results from the serving cluster. + +{{< tabs >}} +{{< tab "View manager role" >}} +{{< include-example file="examples/rbac/create_roles" +example="create-role-view-manager" >}} +{{}} +{{< tab "Serving index manager role" >}} +{{< include-example file="examples/rbac/create_roles" +example="create-role-serving-index-manager" >}} +{{}} +{{< tab "Data reader role" >}} +{{< include-example file="examples/rbac/create_roles" example="create-role-data-reader">}} +{{}} +{{}} + +In Materialize, a role is created with inheritance support. With inheritance, +when a role is granted to another role (i.e., the target role), the target role +inherits privileges (not role attributes and parameters) through the other role. +{{< include-md file="shared-content/rbac/db-roles-public-membership.md" >}} + +{{% include-md file="shared-content/rbac/db-roles-managing-privileges.md" %}} + +{{< annotation type="Disambiguation" >}} +{{% include-md file="shared-content/rbac/grant-vs-alter-default-privilege.md" +%}} +{{}} + +See also: + +- For a list of required privileges for specific operations, see [Appendix: +Privileges by command](/manage/access-control/appendix-command-privileges/). + +## Manage current privileges for a role + +{{< note >}} + +- The examples below assume the existence of a `mydb` database and a `sales` +schema within the `mydb` database. + +- The examples below assume the roles only need privileges to objects in the + `mydb.sales` schema. + +{{}} + +### View privileges for a role + +{{< annotation type="Privilege(s) required to run the command" >}} + +No specific privilege is required to run the `SHOW PRIVILEGES` + +{{}} + +To view privileges granted to a role, you can use the [`SHOW +PRIVILEGES`](/sql/show-privileges) command, substituting `` with the role +name (see [`SHOW PRIVILEGES`](/sql/show-default-privileges) for the full +syntax): + +```mzsql +SHOW PRIVILEGES FOR ; +``` + +{{< note >}} +{{< include-md file="shared-content/rbac/db-roles-public-membership.md" >}} +{{}} + +For example: + +{{< tabs >}} +{{< tab "User">}} + +{{< include-example file="examples/rbac/show_privileges" +example="for-user">}} + +{{< include-example file="examples/rbac/show_privileges" +example="example-results">}} + +{{}} +{{< tab "Service account role">}} + +{{< include-example file="examples/rbac/show_privileges" +example="for-service-account">}} +{{< include-example file="examples/rbac/show_privileges" +example="example-results">}} +{{}} +{{< tab "Functional roles">}} +{{< tabs >}} +{{< tab "View manager role" >}} +{{< include-example file="examples/rbac/show_privileges" +example="for-view-manager" >}} +{{< include-example file="examples/rbac/show_privileges" +example="example-results">}} +{{}} +{{< tab "Serving index manager role" >}} +{{< include-example file="examples/rbac/show_privileges" +example="for-serving-index-manager" >}} +{{< include-example file="examples/rbac/show_privileges" +example="example-results">}} +{{}} +{{< tab "Data reader role" >}} +{{< include-example file="examples/rbac/show_privileges" +example="for-data-reader">}} +{{< include-example file="examples/rbac/show_privileges" +example="example-results">}} +{{}} +{{}} + +{{}} +{{}} + +{{< tip >}} + +For the `SHOW PRIVILEGES` command, you can add a `WHERE` clause to filter by the +return fields; e.g., `SHOW PRIVILEGES FOR view_manager WHERE +name='quickstart';`. + +{{}} + +### Grant privileges to a role + +To grant [privileges](/manage/access-control/appendix-command-privileges/) to +a role, use the [`GRANT PRIVILEGE`](/sql/grant-privilege/) statement (see +[`GRANT PRIVILEGE`](/sql/grant-privilege/) for the full syntax) + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/grant-privilege.md" +>}} + +To override the **object ownership** requirements to grant privileges, run as a +user with superuser privileges; e.g. `mz_system` user. + +{{}} + +```mzsql +GRANT ON TO ; +``` + +{{< include-md file="shared-content/rbac/use-resusable-roles.md" >}} + +For example, the following grants privileges to the functional roles. + +{{< note >}} +{{< include-md file="shared-content/rbac/privileges-related-objects.md" >}} +{{}} + +{{< tabs >}} +{{< tab "View manager role" >}} +{{< include-example file="examples/rbac/grant_privileges" +example="to-view-manager" >}} + +{{< include-example file="examples/rbac/grant_privileges" +example="to-view-manager-results" >}} +{{}} +{{< tab "Serving index manager role" >}} +{{< include-example file="examples/rbac/grant_privileges" +example="to-serving-index-manager" >}} + +{{< include-example file="examples/rbac/grant_privileges" +example="to-serving-index-manager-results" >}} +{{}} +{{< tab "Data reader role" >}} +{{< include-example file="examples/rbac/grant_privileges" +example="to-data-reader">}} + +{{< include-example file="examples/rbac/grant_privileges" example="to-data-reader-results">}} +{{}} +{{}} + +### Grant a role to another role + +Once a role is created, you can modify its privileges either: + +- Directly by [granting privileges for a role](#grant-privileges-to-a-role) or + [revoking privileges from a role](#revoke-privileges-from-a-role). +- Indirectly (through inheritance) by granting other roles to the role or + [revoking roles from the role](#revoke-a-role-from-another-role). + +{{< tip >}} + +{{< include-md file="shared-content/rbac/use-resusable-roles.md" >}} + +{{}} + +To grant a role to another role (where the role can be a user role/service +account role/functional role), use the [`GRANT ROLE`](/sql/grant-role/) +statement (see [`GRANT ROLE`](/sql/grant-role/) for full syntax): + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/grant-role.md" +>}} + +`mz_system` user has the required privileges on the system. +{{}} + +```mzsql +GRANT [, ...] to [, ...]; +``` + +When a role is granted to another role, the target role becomes a member of the +other role and inherits the privileges through the other role. + +In the following examples, + +- The functional role `view_manager` is granted to the user role + `blue.berry@example.com`. +- The functional role `serving_index_manager` is granted to the functional role + `view_manager`. +- The functional role `data_reader` is granted to the service account role + `sales_report_app`. + +{{< tabs >}} +{{< tab "Grant view_manager role" >}} + +{{< include-example file="examples/rbac/grant_roles" +example="view_manager" >}} + +{{< include-example file="examples/rbac/grant_roles" +example="view_manager-results-show-privileges" >}} + +{{< include-example file="examples/rbac/grant_roles" +example="view_manager-results-create-objects" >}} + +{{}} + +{{< tab "Grant serving_index_manager role" >}} +{{< include-example file="examples/rbac/grant_roles" +example="serving_index_manager" >}} + +Review the privileges of `view_manager` as well as `"blue.berry@example.com"` +(a member of `view_manager`) after the grant. + +{{< tabs >}} +{{< tab "Privileges for view_manager">}} +{{< include-example file="examples/rbac/grant_roles" +example="serving_index_manager-results-show-privileges-view_manager" +>}} + +{{}} +{{< tab "Privileges for blue.berry@example.com">}} + +{{< include-example file="examples/rbac/grant_roles" +example="serving_index_manager-results-show-privileges-view_manager-member" +>}} + +{{}} +{{}} + +{{< include-example file="examples/rbac/grant_roles" +example="serving_index_manager-results-show-privileges-results" +>}} + +{{< include-example file="examples/rbac/grant_roles" +example="serving_index_manager-results-create-index" >}} + +{{}} + +{{< tab "Grant data_reader role" >}} +{{< include-example file="examples/rbac/grant_roles" +example="data_reader" >}} + +{{< include-example file="examples/rbac/grant_roles" +example="data_reader-results-show-privileges" >}} + +{{< include-example file="examples/rbac/grant_roles" +example="data_reader-results-select" >}} + +{{}} +{{}} + +### Revoke privileges from a role + +To remove privileges from a role, use the [`REVOKE `](/sql/revoke-privilege/) statement: + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/revoke-privilege.md" +>}} + +{{}} + +```mzsql +REVOKE ON FROM ; +``` + +### Revoke a role from another role + +To revoke a role from another role, use the [`REVOKE `](/sql/revoke-role/) statement: + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/revoke-role.md" +>}} + +{{}} + +```mzsql +REVOKE FROM ; +``` + +For example: + +```mzsql +REVOKE data_reader FROM sales_report_app; +``` + +{{< important >}} +{{< include-md file="shared-content/rbac/revoke-roles-consideration.md" >}} +{{}} + +## Manage future privileges for a role + +In Materialize, a role automatically gets all [applicable +privileges](/manage/access-control/appendix-privileges/) for an object they +create/own; for example, the creator of a schema gets `CREATE` and `USAGE`; the +creator of a table gets `SELECT`, `INSERT`, `UPDATE`, and `DELETE`. However, for +others to access the new object, you can either manually grant privileges on new +objects or use default privileges to automatically grant privileges to others as +new objects are created. + +Default privileges can be specified for a given object type and scoped to: + +- all future objects of that type; +- all future objects of that type within specific databases or schemas; +- all future objects of that type created by specific roles (or by all roles + `PUBLIC`). + +{{< include-md file="shared-content/rbac/default-privilege-clarification.md" >}} + +{{< annotation type="Disambiguation" >}} +{{% include-md file="shared-content/rbac/grant-vs-alter-default-privilege.md" +%}} + +{{}} + +### View default privileges + +To view default privileges, you can use the [`SHOW DEFAULT +PRIVILEGES`](/sql/show-default-privileges) command, substituting `` with +the role name (see [`SHOW DEFAULT PRIVILEGES`](/sql/show-default-privileges) for +the full syntax): + +{{< annotation type="Privilege(s) required to run the command" >}} + +No specific privilege is required to run the `SHOW DEFAULT PRIVILEGES`. + +{{}} + +```mzsql +SHOW DEFAULT PRIVILEGES FOR ; +``` + +For example: + +{{< tabs >}} +{{< tab "User">}} + +{{< include-example file="examples/rbac/show_default_privileges" +example="for-user">}} + +{{}} +{{< tab "Service account role">}} + +{{< include-example file="examples/rbac/show_default_privileges" +example="for-service-account">}} + +{{}} +{{< tab "Functional roles">}} +{{< tabs >}} +{{< tab "View manager role" >}} +{{< include-example file="examples/rbac/show_default_privileges" +example="for-view-manager" >}} +{{}} +{{< tab "Serving index manager role" >}} +{{< include-example file="examples/rbac/show_default_privileges" +example="for-serving-index-manager" >}} +{{}} +{{< tab "Data reader role" >}} +{{< include-example file="examples/rbac/show_default_privileges" example="for-data-reader">}} +{{}} +{{}} + +{{}} +{{}} + +### Alter default privileges + +To define default privilege for objects created by a role, use the [`ALTER +DEFAULT PRIVILEGES`](/sql/alter-default-privileges) command (see [`ALTER +DEFAULT PRIVILEGES`](/sql/alter-default-privileges) for the full syntax): + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md +file="shared-content/sql-command-privileges/alter-default-privileges.md" +>}} + +{{}} + +```mzsql +ALTER DEFAULT PRIVILEGES FOR ROLE + IN SCHEMA -- Optional. If specified, need USAGE on database and schema. + GRANT ON TO ; +``` + +{{< note >}} +- With the exception of the `PUBLIC` role, the `` role is + **not** transitive. That is, default privileges that specify a functional role + like `view_manager` as the `` do **not** apply to objects + created by its members. + + However, you can approximate default privileges for a functional role by + restricting `CREATE` privileges for the objects to the desired functional + roles (e.g., only `view_managers` have privileges to create tables in + `mydb.sales` schema) and then specify `PUBLIC` as the ``. + +- As with any other grants, the privileges granted to the `` are + inherited by the members of the ``. + +{{}} + +{{< tabs >}} +{{< tab "Specify blue.berry as the object creator" >}} +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-blueberry" %}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-blueberry-verification-setup" %}} + +To verify that the default privileges have been automatically granted, you can +run `SHOW PRIVILEGES`: + +{{< tabs >}} + +{{< tab "view_manager">}} +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-blueberry-verification-view_manager" %}} +{{}} +{{< tab "data_reader" >}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-blueberry-verification-data_reader" %}} + +{{}} +{{< tab "sales_report_app (a member of data_reader)" >}} +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-blueberry-verification-sales_report_app" %}} +{{}} +{{}} + + +{{}} +{{< tab "Specify PUBLIC as the object creator" >}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-view-manager" %}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-view-manager-member" %}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-public" %}} + + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-public-member" %}} + +To verify that the default privileges have been automatically granted to others, +you can run `SHOW PRIVILEGES`: + +{{< tabs >}} +{{< tab "view_manager">}} + +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-public-member-verification-view_manager" %}} +{{}} +{{< tab "blue.berry@example.com" >}} +{{% include-example file="examples/rbac/alter_default_privileges" +example="for-tables-created-by-public-member-verification-blueberry" %}} + +{{}} +{{}} +{{}} +{{}} +## Show roles in system + +To view the roles in the system, use the [`SHOW ROLES`](/sql/show-roles/) command: + +```mzsql +SHOW ROLES [ LIKE | WHERE ]; +``` + +{{% include-example file="examples/rbac/show_roles" example="all-roles" %}} + +## Drop a role + +To remove a role from the system, use the [`DROP ROLE`](/sql/drop-role/) +command: + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/drop-role.md" +>}} + +{{}} + +```mzsql +DROP ROLE ; +``` + +{{< note >}} +You cannot drop a role if it contains any members. Before dropping a role, +revoke the role from all its members. See [Revoke a role](#revoke-a-role-from-another-role). +{{}} + + +## Alter role + +When granting privileges, the privileges may be scoped to a particular cluster, +database, and schema. + +You can use [`ALTER ROLE ... SET`](/sql/alter-role/) to set various +configuration parameters, including cluster, database, and schema. + +```mzsql +ALTER ROLE SET =|TO ; +``` +{{% include-example file="examples/rbac/alter_roles" +example="alter-roles-configurations" %}} + +{{% include-example file="examples/rbac/alter_roles" +example="alter-roles-configs-not-inherited" %}} + +## Change ownership of objects + +Certain [commands on an +object](/manage/access-control/appendix-command-privileges/) (such as creating +an index on a materialized view or changing owner of an object) require +ownership of the object itself (or *superuser* privileges). + +In Materialize, when a role creates an object, the role becomes the owner of the +object and is automatically granted all [applicable +privileges](/manage/access-control/appendix-privileges/) for the object. To +transfer ownership (and privileges) to another role (another user role/service +account role/functional role), you can use the [ALTER ... OWNER +TO](/sql/alter-owner/) command: + +{{< annotation type="Privilege(s) required to run the command" >}} + +{{< include-md file="shared-content/sql-command-privileges/alter-owner.md" +>}} + +{{}} + +```mzsql +ALTER OWNER TO ; +``` + +Before changing the ownership, review the privileges of the current owner +(`lemon@example.com`) and the future owner (`view_manage`): + +{{% include-example file="examples/rbac/alter_owner" +example="view-privileges-for-current-owner" %}} + +{{% include-example file="examples/rbac/alter_owner" +example="view-privileges-for-future-owner" %}} + +{{% include-example file="examples/rbac/alter_owner" +example="table-shared_lemon" %}} + +{{% include-example file="examples/rbac/alter_owner" +example="view-privileges-for-new-owner" %}} + +{{% include-example file="examples/rbac/alter_owner" +example="view-privileges-for-previous-owner" %}} + +## See also + +- [Access control best practices](/manage/access-control/#best-practices) +- [Manage privileges with + Terraform](/manage/access-control/rbac-terraform-tutorial/) diff --git a/doc/user/content/self-managed/_index.md b/doc/user/content/self-managed/_index.md index c2ede8fb60ce6..65d6b24dc727b 100644 --- a/doc/user/content/self-managed/_index.md +++ b/doc/user/content/self-managed/_index.md @@ -3,8 +3,50 @@ title: "Self-managed Materialize" description: "" aliases: - /self-hosted/ -robots: "noindex, nofollow" -draft: true +menus: + main: + weight: 3 + identifier: 'self-managed' +disable_list: true --- - +Materialize is a real-time data integration platform that enables you to use SQL +to transform, deliver, and act on fast changing data. + +## Self-managed Materialize + +With self-managed Materialize, you can deploy and operate Materialize in your +Kubernetes environment. + +{{% self-managed/self-managed-editions-table %}} + +{{< callout >}} + +{{< self-managed/also-available >}} + +- [Materialize MCP Server](https://materialize.com/blog/materialize-turns-views-into-tools-for-agents/). + The Materialize MCP Server bridges SQL and AI by transforming indexed views into well-typed tools + that agents can call directly. It lets you expose your freshest, most complex logic as operational + data products automatically and reliably. For more information, see the + [Materialize MCP Server](/integrations/llm/) documentation. + +{{}} + +## Requirements + +The self-managed Materialize requires the following: + +{{% self-managed/materialize-components-list %}} + +See the [Installation guide](./installation/) for more information. + +## Installation + +For instructions on installing Materialize on your Kubernetes cluster, see: + +- [Install locally on kind](./installation/install-on-local-kind/) +- [Install locally on + minikube](./installation/install-on-local-minikube/) +- [Install on AWS](./installation/install-on-aws/) +- [Install on GCP](./installation/install-on-gcp/) +- [Install on Azure](./installation/install-on-azure/) diff --git a/doc/user/content/self-managed/installation/_index.md b/doc/user/content/self-managed/installation/_index.md new file mode 100644 index 0000000000000..44a612fa0f3fb --- /dev/null +++ b/doc/user/content/self-managed/installation/_index.md @@ -0,0 +1,69 @@ +--- +title: "Installation" +description: "Installation guides for self-managed Materialize." +disable_list: true +menu: + main: + parent: self-managed + identifier: "installation" + weight: 5 +--- + +You can install self-managed Materialize on a Kubernetes cluster running locally +or on a cloud provider. Self-managed Materialize requires: + +{{% self-managed/materialize-components-list %}} + +## Install locally + +{{< multilinkbox >}} +{{< linkbox title="Using Docker/kind" icon="materialize">}} +[Install locally on kind](/self-managed/installation/install-on-local-kind/) +{{}} +{{< linkbox title="Using Docker/minikube" icon="materialize">}} +[Install locally on minikube](/self-managed/installation/install-on-local-minikube/) +{{}} +{{}} + +## Install on cloud provider + +{{< multilinkbox >}} + +{{< linkbox title="[Install on AWS](/self-managed/installation/install-on-aws/)" icon="materialize">}} + +[Deploy Materialize to AWS Elastic Kubernetes Service (EKS)](/self-managed/installation/install-on-aws/) + +{{}} + +{{< linkbox title="[Install on Azure](/self-managed/installation/install-on-azure/)" icon="materialize">}} + +[Deploy Materialize to Azure Kubernetes Service (AKS)](/self-managed/installation/install-on-azure/) + +{{}} + +{{< linkbox icon="materialize" title="[Install on GCP](/self-managed/installation/install-on-gcp/)" >}} + +[Deploy Materialize to Google Kubernetes Engine (GKE)](/self-managed/installation/install-on-gcp/) +{{}} + +{{}} + +See also: + +- [AWS Deployment + guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/#recommended-instance-types) + +- [GCP Deployment + guidelines](/self-managed/installation/install-on-gcp/appendix-deployment-guidelines/#recommended-instance-types) + +- [Azure Deployment + guidelines](/self-managed/installation/install-on-azure/appendix-deployment-guidelines/#recommended-instance-types) + +{{< callout >}} +{{< self-managed/also-available >}} +{{}} + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/installation/appendix-aws-production-guide.md b/doc/user/content/self-managed/installation/appendix-aws-production-guide.md new file mode 100644 index 0000000000000..2049efd0a2543 --- /dev/null +++ b/doc/user/content/self-managed/installation/appendix-aws-production-guide.md @@ -0,0 +1,19 @@ +--- +title: "Production guide" +description: "Production guide for Self-Managed Materialize AWS deployment." +draft: true + +--- + +The sample Materialize on AWS Terraform module are provided as a starting point +for evaluation. The Terraform module: + +- Does not use encryption for object storage. +- Does not use TLS for traffic. +- Does not implement RBAC (Role-Based Access Control). +- Does not include explicit OS version upgrade logic. +- Does not include Kubernetes upgrade logic. +- Relies on port forwarding for access. + + + diff --git a/doc/user/content/self-managed/installation/appendix-terraforms.md b/doc/user/content/self-managed/installation/appendix-terraforms.md new file mode 100644 index 0000000000000..612f38edfa562 --- /dev/null +++ b/doc/user/content/self-managed/installation/appendix-terraforms.md @@ -0,0 +1,53 @@ + +--- +title: "Appendix: Terraforms" +description: "List of template Terraform modules that are available as a +starting point." +menu: + main: + parent: "installation" + identifier: "appendix-terraforms" + weight: 100 +--- + +To help you get started, Materialize provides some template Terraforms. + +{{< important >}} +These modules are intended for evaluation/demonstration purposes and for serving +as a template when building your own production deployment. The modules should +not be directly relied upon for production deployments: **future releases of the +modules will contain breaking changes.** Instead, to use as a starting point for +your own production deployment, either: + +- Fork the repo and pin to a specific version; or + +- Use the code as a reference when developing your own deployment. + +{{}} + +{{< yaml-table data="self_managed/terraform_list" >}} + +## Releases + +### Materialize on AWS Terraform module + +{{< yaml-table data="self_managed/aws_terraform_versions" >}} + +{{% self-managed/aws-terraform-upgrade-notes %}} + +See also [Upgrade Notes]( +https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#upgrade-notes) +for release-specific upgrade notes. + + +### Materialize on Azure Terraform module + +{{< yaml-table data="self_managed/azure_terraform_versions" >}} + +{{% self-managed/azure-terraform-upgrade-notes %}} + +### Materialize on GCP Terraform module + +{{< yaml-table data="self_managed/gcp_terraform_versions" >}} + +{{% self-managed/gcp-terraform-upgrade-notes %}} diff --git a/doc/user/content/self-managed/installation/configuration.md b/doc/user/content/self-managed/installation/configuration.md new file mode 100644 index 0000000000000..f350a0d440789 --- /dev/null +++ b/doc/user/content/self-managed/installation/configuration.md @@ -0,0 +1,43 @@ +--- +title: "Materialize Operator Configuration" +description: "Configuration reference for the Materialize Operator Helm chart" +aliases: + - /self-hosted/configuration/ +menu: + main: + parent: "installation" + weight: 70 +--- + +## Configure the Materialize operator + +To configure the Materialize operator, you can: + +- Use a configuration YAML file (e.g., `values.yaml`) that specifies the + configuration values and then install the chart with the `-f` flag: + + ```shell + # Assumes you have added the Materialize operator Helm chart repository + helm install my-materialize-operator materialize/materialize-operator \ + -f /path/to/your/config/values.yaml + ``` + +- Specify each parameter using the `--set key=value[,key=value]` argument to + `helm install`. For example: + + ```shell + # Assumes you have added the Materialize operator Helm chart repository + helm install my-materialize-operator materialize/materialize-operator \ + --set observability.podMetrics.enabled=true + ``` + +{{% self-managed/materialize-operator-chart-parameters-table %}} + +## Parameters + +{{% self-managed/materialize-operator-chart-parameters %}} + +## See also + +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/installation/ingress-aws.md b/doc/user/content/self-managed/installation/ingress-aws.md new file mode 100644 index 0000000000000..0900ff0ff8efe --- /dev/null +++ b/doc/user/content/self-managed/installation/ingress-aws.md @@ -0,0 +1,7 @@ +--- +title: "Ingress guide: AWS" +description: "How to access Self-managed Materialize deployment on AWS" +draft: true +--- + + diff --git a/doc/user/content/self-managed/installation/ingress-azure.md b/doc/user/content/self-managed/installation/ingress-azure.md new file mode 100644 index 0000000000000..ffbbda57b6d0f --- /dev/null +++ b/doc/user/content/self-managed/installation/ingress-azure.md @@ -0,0 +1,8 @@ +--- +title: "Ingress guide: Azure" +description: "How to access Self-managed Materialize deployment on Azure" +draft: true +--- + + + diff --git a/doc/user/content/self-managed/installation/ingress-google.md b/doc/user/content/self-managed/installation/ingress-google.md new file mode 100644 index 0000000000000..d41fa6d0d970f --- /dev/null +++ b/doc/user/content/self-managed/installation/ingress-google.md @@ -0,0 +1,7 @@ +--- +title: "Ingress guide: GCP" +description: "How to access Self-managed Materialize deployment on Google Cloud" +draft: true +--- + + diff --git a/doc/user/content/self-managed/installation/install-on-aws/_index.md b/doc/user/content/self-managed/installation/install-on-aws/_index.md new file mode 100644 index 0000000000000..ad6a5eff979b1 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-aws/_index.md @@ -0,0 +1,477 @@ +--- +title: "Install on AWS (via Terraform)" +description: "Install Materialize on AWS Elastic Kubernetes Service (EKS) using Terraform." +aliases: + - /self-hosted/install-on-aws/ +disable_list: true +menu: + main: + parent: "installation" + identifier: "install-on-aws" + weight: 20 +--- + +{{% self-managed/materialize-components-sentence %}} + +The tutorial deploys Materialize to AWS Elastic Kubernetes Service (EKS) with a +PostgreSQL RDS database as the metadata database and AWS S3 for blob storage. +The tutorial uses [Materialize on AWS Terraform +module](https://github.com/MaterializeInc/terraform-aws-materialize) to: + +- Set up the AWS Kubernetes environment. +- Call + [terraform-helm-materialize](https://github.com/MaterializeInc/terraform-helm-materialize) + module to deploy Materialize Operator and Materialize instances to that EKS + cluster. + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< /warning >}} + +{{% self-managed/aws-recommended-instances %}} + +See [Appendix: AWS Deployment +guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/) for +more information. + +## Prerequisites + +### Terraform + +If you don't have Terraform installed, [install +Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### AWS CLI + +If you do not have the AWS CLI installed, install. For details, see the [AWS +documentation](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). + +### kubectl + +If you do not have `kubectl`, install. See the [Amazon EKS: install `kubectl` +documentation](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) +for details. + +### Helm 3.2.0+ + +If you do not have Helm 3.2.0+, install. For details, see the [Helm +documentation](https://helm.sh/docs/intro/install/). + +## Set up AWS Kubernetes environment and install Materialize + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< self-managed/tutorial-disclaimer >}} + +{{< /warning >}} + +{{< tabs >}} + +{{< tab "Deployed components" >}} + +[Materialize on AWS Terraform +module](https://github.com/MaterializeInc/terraform-aws-materialize/blob/main/README.md) +deploys a sample infrastructure on AWS (region `us-east-1`) with the following +components: + +{{< yaml-table data="self_managed/aws_terraform_deployed_components" >}} + +{{< tip >}} +{{% self-managed/aws-terraform-configs %}} +{{< /tip >}} + +{{}} +{{< tab "Releases" >}} + +{{< yaml-table data="self_managed/aws_terraform_versions" >}} + +{{}} +{{}} + +1. Open a Terminal window. + +1. Configure AWS CLI with your AWS credentials. For details, see the [AWS + documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). + +{{% self-managed/versions/step-clone-aws-terraform-repo %}} + +1. Go to the `examples/simple` folder in the Materialize Terraform repo + directory. + + ```bash + cd terraform-aws-materialize/examples/simple + ``` + + {{< tip >}} + {{< self-managed/aws-terraform-configs >}} + {{< /tip >}} + +1. Create a `terraform.tfvars` file (you can copy from the + `terraform.tfvars.example` file) and specify the following variables: + + | Variable | Description | + |--------------------|-------------| + | `namespace` | A namespace (e.g., `my-demo`) that will be used to form part of the prefix for your AWS resources.
**Requirements:**
- Maximum of 12 characters
- Must start with a lowercase letter
- Must be lowercase alphanumeric and hyphens only | + | `environment` | An environment name (e.g., `dev`, `test`) that will be used to form part of the prefix for your AWS resources.
**Requirements:**
- Maximum of 8 characters
- Must be lowercase alphanumeric only | + + + ```bash + # The namespace and environment variables are used to construct the names of the resources + # e.g. ${namespace}-${environment}-storage, ${namespace}-${environment}-db etc. + + namespace = "enter-namespace" // maximum 12 characters, start with a letter, contain lowercase alphanumeric and hyphens only (e.g. my-demo) + environment = "enter-environment" // maximum 8 characters, lowercase alphanumeric only (e.g., dev, test) + ``` + + {{< tip >}} + {{< self-managed/aws-terraform-configs >}} + {{< /tip >}} + +1. Initialize the terraform directory. + + ```bash + terraform init + ``` + +1. Use terraform plan to review the changes to be made. + + ```bash + terraform plan + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply + ``` + + To approve the changes and apply, enter `yes`. + + + + Upon successful completion, various fields and their values are output: + + ```none + Apply complete! Resources: 89 added, 0 changed, 0 destroyed. + + Outputs: + + cluster_certificate_authority_data = + database_endpoint = "my-demo-dev-db.abcdefg8dsto.us-east-1.rds.amazonaws.com:5432" + eks_cluster_endpoint = "https://0123456789A00BCD000E11BE12345A01.gr7.us-east-1.eks.amazonaws.com" + eks_cluster_name = "my-demo-dev-eks" + materialize_s3_role_arn = "arn:aws:iam::000111222333:role/my-demo-dev-mz-role" + metadata_backend_url = + nlb_details = [] + oidc_provider_arn = "arn:aws:iam::000111222333:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/7D14BCA3A7AA896A836782D96A24F958" + persist_backend_url = "s3://my-demo-dev-storage-f2def2a9/dev:serviceaccount:materialize-environment:12345678-1234-1234-1234-12345678912" + s3_bucket_name = "my-demo-dev-storage-f2def2a9" + vpc_id = "vpc-0abc000bed1d111bd" + ``` + +1. Note your specific values for the following fields: + + - `eks_cluster_name` (Used to configure `kubectl`) + +1. Configure `kubectl` to connect to your EKS cluster, replacing: + + - `` with the name of your EKS cluster. Your cluster + name has the form `{namespace}-{environment}-eks`; e.g., + `my-demo-dev-eks`. + + - `` with the region of your EKS cluster. The + simple example uses `us-east-1`. + + ```bash + aws eks update-kubeconfig --name --region + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl get nodes + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +1. By default, the example Terraform installs the Materialize Operator and, + starting in v0.4.0, a `cert-manager`. Verify the installation and check the + status: + + {{< tabs >}} + {{< tab "Materialize Operator" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/my-demo-dev-materialize-operator-84ff4b4648-brjhl 1/1 Running 0 12s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/my-demo-dev-materialize-operator 1/1 1 1 12s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/my-demo-dev-materialize-operator-84ff4b4648 1 1 1 12s + ``` + + {{}} + {{< tab "cert-manager (Starting in version 0.4.0)" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n cert-manager + ``` + Wait for the components to be in the `Running` state: + ``` + NAME READY STATUS RESTARTS AGE + pod/cert-manager-cainjector-686546c9f7-v9hwp 1/1 Running 0 4m20s + pod/cert-manager-d6746cf45-cdmb5 1/1 Running 0 4m20s + pod/cert-manager-webhook-5f79cd6f4b-rcjbq 1/1 Running 0 4m20s + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/cert-manager ClusterIP 172.20.2.136 9402/TCP 4m20s + service/cert-manager-cainjector ClusterIP 172.20.154.137 9402/TCP 4m20s + service/cert-manager-webhook ClusterIP 172.20.63.217 443/TCP,9402/TCP 4m20s + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/cert-manager 1/1 1 1 4m20s + deployment.apps/cert-manager-cainjector 1/1 1 1 4m20s + deployment.apps/cert-manager-webhook 1/1 1 1 4m20s + NAME DESIRED CURRENT READY AGE + replicaset.apps/cert-manager-cainjector-686546c9f7 1 1 1 4m20s + replicaset.apps/cert-manager-d6746cf45 1 1 1 4m20s + replicaset.apps/cert-manager-webhook-5f79cd6f4b 1 1 1 + 4m20s + ``` + + {{}} + {{}} + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting) guide. + +1. Once the Materialize operator is deployed and running, you can deploy the + Materialize instances. To deploy Materialize instances, create a + `mz_instances.tfvars` file with the [Materialize instance + configuration](https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_materialize_instances). + + For example, the following specifies the configuration for a `demo` instance. + + ```bash + cat < mz_instances.tfvars + + materialize_instances = [ + { + name = "demo" + namespace = "materialize-environment" + database_name = "demo_db" + cpu_request = "1" + memory_request = "2Gi" + memory_limit = "2Gi" + } + ] + EOF + ``` + + - **Starting in v0.3.0**, the Materialize on AWS Terraform module also + deploys, by default, Network Load Balancers (NLBs) for each Materialize + instance (i.e., the + [`create_nlb`](https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_materialize_instances) + flag defaults to `true`). The NLBs, by default, are configured to be + internal (i.e., the + [`internal_nlb`](https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_materialize_instances) + flag defaults to `true`). See [`materialize_instances`]( + https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_materialize_instances) + for the Materialize instance configuration options. + + - **Starting in v0.4.0**, a self-signed `ClusterIssuer` is deployed by + default. The `ClusterIssuer` is deployed on subsequent after the + `cert-manager` is running. + + - **Starting in v0.4.6**, you can specify addition configuration options via + `environmentd_extra_args`. + + {{< tip >}} + {{% self-managed/aws-terraform-upgrade-notes %}} + + See [Materialize on AWS releases](/self-managed/installation/appendix-terraforms/#materialize-on-aws-terraform-module) for notable changes. + {{}} + +1. Run `terraform plan` with both `.tfvars` files and review the changes to be + made. + + ```bash + terraform plan -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + The plan should show the changes to be made, with a summary similar to the + following: + + ``` + Plan: 17 to add, 1 to change, 0 to destroy. + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + To approve the changes and apply, enter `yes`. + + Upon successful completion, you should see output with a summary similar to + the following: + + + + ```bash + Apply complete! Resources: 17 added, 1 changed, 0 destroyed. + + Outputs: + + cluster_certificate_authority_data = + database_endpoint = "my-demo-dev-db.abcdefg8dsto.us-east-1.rds.amazonaws.com:5432" + eks_cluster_endpoint = "https://0123456789A00BCD000E11BE12345A01.gr7.us-east-1.eks.amazonaws.com" + eks_cluster_name = "my-demo-dev-eks" + materialize_s3_role_arn = "arn:aws:iam::000111222333:role/my-demo-dev-mz-role" + metadata_backend_url = + nlb_details = [ + "demo" = { + "arn" = "arn:aws:elasticloadbalancing:us-east-1:000111222333:loadbalancer/net/my-demo-dev/aeae3d936afebcfe" + "dns_name" = "my-demo-dev-aeae3d936afebcfe.elb.us-east-1.amazonaws.com" + } + ] + oidc_provider_arn = "arn:aws:iam::000111222333:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/7D14BCA3A7AA896A836782D96A24F958" + persist_backend_url = "s3://my-demo-dev-storage-f2def2a9/dev:serviceaccount:materialize-environment:12345678-1234-1234-1234-12345678912" + s3_bucket_name = "my-demo-dev-storage-f2def2a9" + vpc_id = "vpc-0abc000bed1d111bd" + ``` + + The Network Load Balancer (NLB) details `nlb_details` are available when + running the Terraform module v0.3.0+. + +1. Verify the installation and check the status: + + ```bash + kubectl get all -n materialize-environment + ``` + + Wait for the components to be in the `Running` state. + + ```none + NAME READY STATUS RESTARTS AGE + pod/create-db-demo-db-6swk7 0/1 Completed 0 33s + pod/mzutd2fbabf5-balancerd-6c9755c498-28kcw 1/1 Running 0 11s + pod/mzutd2fbabf5-cluster-s2-replica-s1-gen-1-0 1/1 Running 0 11s + pod/mzutd2fbabf5-cluster-u1-replica-u1-gen-1-0 1/1 Running 0 11s + pod/mzutd2fbabf5-console-57f94b4588-6lg2x 1/1 Running 0 4s + pod/mzutd2fbabf5-console-57f94b4588-v65lk 1/1 Running 0 4s + pod/mzutd2fbabf5-environmentd-1-0 1/1 Running 0 16s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/mzutd2fbabf5-balancerd ClusterIP None 6876/TCP,6875/TCP 11s + service/mzutd2fbabf5-cluster-s2-replica-s1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 12s + service/mzutd2fbabf5-cluster-u1-replica-u1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 12s + service/mzutd2fbabf5-console ClusterIP None 8080/TCP 4s + service/mzutd2fbabf5-environmentd ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 11s + service/mzutd2fbabf5-environmentd-1 ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 16s + service/mzutd2fbabf5-persist-pubsub-1 ClusterIP None 6879/TCP 16s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/mzutd2fbabf5-balancerd 1/1 1 1 11s + deployment.apps/mzutd2fbabf5-console 2/2 2 2 4s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/mzutd2fbabf5-balancerd-6c9755c498 1 1 1 11s + replicaset.apps/mzutd2fbabf5-console-57f94b4588 2 2 2 4s + + NAME READY AGE + statefulset.apps/mzutd2fbabf5-cluster-s2-replica-s1-gen-1 1/1 12s + statefulset.apps/mzutd2fbabf5-cluster-u1-replica-u1-gen-1 1/1 11s + statefulset.apps/mzutd2fbabf5-environmentd-1 1/1 16s + + NAME STATUS COMPLETIONS DURATION AGE + job.batch/create-db-demo-db Complete 1/1 11s 33s + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting/). + +1. Open the Materialize Console in your browser: + + {{< tabs >}} + + {{< tab "Via Network Load Balancer" >}} + + Starting in v0.3.0, for each Materialize instance, Materialize on AWS + Terraform module also deploys AWS Network Load Balancers (by default, + internal) with the following listeners, including a listener on port 8080 for + the Materialize Console: + + | Port | Description | + | ---- | ------------| + | 6875 | For SQL connections to the database | + | 6876 | For HTTP(S) connections to the database | + | **8080** | **For HTTP(S) connections to Materialize Console** | + + The Network Load Balancer (NLB) details are found in the `nlb_details` in + the [Terraform output](#aws-terrafrom-output). + + The example uses a self-signed ClusterIssuer. As such, you may encounter a + warning with regards to the certificate. In production, run with certificates + from an official Certificate Authority (CA) rather than self-signed + certificates. + + {{}} + + {{< tab "Via port forwarding" >}} + + {{% self-managed/port-forwarding-handling %}} + + {{}} + {{}} + + {{< tip >}} + + {{% self-managed/troubleshoot-console-mz_catalog_server_blurb %}} + + {{< /tip >}} + +## Next steps + +{{% self-managed/next-steps %}} + +## Cleanup + +{{% self-managed/cleanup-cloud %}} + + {{< tip >}} + + - To delete your S3 bucket, you may need to empty the S3 bucket first. If the + `terraform destroy` command is unable to delete the S3 bucket and does not + progress beyond "Still destroying...", empty the S3 bucket first and rerun + the `terraform destroy` command. + + - Upon successful destroy, you may receive some informational messages with + regards to CustomResourceDefinition(CRD). You may safely ignore these + messages as your whole deployment has been destroyed, including the CRDs. + + {{}} + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Appendix: AWS Deployment +guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-aws/appendix-aws-configuration.md b/doc/user/content/self-managed/installation/install-on-aws/appendix-aws-configuration.md new file mode 100644 index 0000000000000..36774f0906c84 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-aws/appendix-aws-configuration.md @@ -0,0 +1,60 @@ +--- +title: "Appendix: Required configuration" +description: "Required configuration for Materialize on AWS Terraform." +menu: + main: + parent: "install-on-aws" + identifier: "appendix-aws-provider-config" + weight: 50 +--- + +## Required variables + +The following variables are required when using the [Materialize on AWS +Terraform modules](https://github.com/MaterializeInc/terraform-aws-materialize): + +{{< yaml-table data="self_managed/aws_required_variables" >}} + +For a list of all variables, see the +[README.md](https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#inputs) +or the [`variables.tf` +file](https://github.com/MaterializeInc/terraform-aws-materialize/blob/main/variables.tf). + +## Required providers + +Starting in [Materialize on AWS Terraform module +v0.3.0](https://github.com/MaterializeInc/terraform-aws-materialize), you need +to declare the following providers: + +```hcl +provider "aws" { + region = "us-east-1" # or some other region + # Specify additional AWS provider configuration as needed +} + +# Required for EKS authentication +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + command = "aws" + } +} + +# Required for Materialize Operator installation +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + command = "aws" + } + } +} +``` diff --git a/doc/user/content/self-managed/installation/install-on-aws/appendix-deployment-guidelines.md b/doc/user/content/self-managed/installation/install-on-aws/appendix-deployment-guidelines.md new file mode 100644 index 0000000000000..3396f3fe15e84 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-aws/appendix-deployment-guidelines.md @@ -0,0 +1,85 @@ +--- +title: "Appendix: AWS deployment guidelines" +description: "AWS environment setup/deployment guidelines" +aliases: + - /self-hosted/operational-guidelines/ +menu: + main: + parent: "install-on-aws" + identifier: "aws-deployment-guidelines" + weight: 40 +--- + +## Recommended instance types + +As a general guideline, we recommend: + +- Processor Type: ARM-based CPU + +- Sizing: + + - If spill-to-disk is not enabled: 1:8 ratio of vCPU to GiB memory + + - If spill-to-disk is enabled (*Recommended*): 1:16 ratio of vCPU to GiB local + instance storage + +{{% self-managed/aws-recommended-instances %}} + +## Locally-attached NVMe storage + +For optimal performance, Materialize requires fast, locally-attached NVMe +storage. Having a locally-attached storage allows Materialize to spill to disk +when operating on datasets larger than main memory as well as allows for a more +graceful degradation rather than OOMing. Network-attached storage (like EBS +volumes) can significantly degrade performance and is not supported. + +*Starting in v0.3.1 of Materialize on AWS Terraform*, disk support (using +OpenEBS and NVMe instance storage) is enabled, by default, for Materialize. With +this change, the Terraform: + +- Installs OpenEBS via Helm; + +- Configures NVMe instance store volumes using a bootstrap script; + +- Creates appropriate storage classes for Materialize. + +Associated with this change, + +- The following configuration options are available: + + - [`enable_disk_support`] + - [`disk_support_config`] + +- The default [`node_group_instance_types`] has changed from `"r8g.2xlarge"` to + `"r7gd.2xlarge"`. See [Recommended instance + types](#recommended-instance-types). + +[enable disk support]: + https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#disk-support-for-materialize + +[`enable_disk_support`]: + https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_enable_disk_support + +[`disk_support_config`]: + https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_disk_support_config + +[`node_group_instance_types`]: + https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_node_group_instance_types + + +## CPU affinity + +It is strongly recommended to enable the Kubernetes `static` [CPU management policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy). +This ensures that each worker thread of Materialize is given exclusively access to a vCPU. Our benchmarks have shown this +to substantially improve the performance of compute-bound workloads. + +## TLS + +When running with TLS in production, run with certificates from an official +Certificate Authority (CA) rather than self-signed certificates. + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/installation/install-on-aws/upgrade-on-aws.md b/doc/user/content/self-managed/installation/install-on-aws/upgrade-on-aws.md new file mode 100644 index 0000000000000..d10ed94c82f2c --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-aws/upgrade-on-aws.md @@ -0,0 +1,126 @@ +--- +title: "Upgrade on AWS" +description: "Procedure to upgrade your Materialize operator and instances running on AWS" +menu: + main: + parent: "install-on-aws" + identifier: "upgrade-on-aws" + weight: 10 +--- + +To upgrade your Materialize instances, upgrade the Materialize operator first +and then the Materialize instances. The following tutorial upgrades your +Materialize deployment running on AWS Elastic Kubernetes Service (EKS). + +The tutorial assumes you have installed Materialize on AWS Elastic Kubernetes +Service (EKS) using the instructions on [Install on +AWS](/self-managed/installation/install-on-aws/) (either from the examples/simple directory +or the root). + +## Version compatibility + + +{{< tabs >}} + +{{< tab "Helm chart releases" >}} + +The following table presents the versions compatibility for the operator and the +applications: + +{{< yaml-table data="self_managed/self_managed_operator_compatibility" >}} + +{{}} + +{{< tab "Materialize on AWS Terraform Releases" >}} + +When upgrading, you may need or want to update your fork of the Terraform module +to upgrade. + +{{< yaml-table data="self_managed/aws_terraform_versions" >}} + +{{}} +{{}} + +## Prerequisites + +{{< important >}} + +The following procedure performs an in-place upgrade, which incurs downtime. + +To perform a rolling upgrade(where both the old and new Materialize instances +are running before the the old instances are removed), you can specify +`inPlaceRollout` to false. When performing a rolling upgrade, ensure you have +enough resources to support having both the old and new Materialize instances +running. + +{{}} + +### Terraform + +If you don't have Terraform installed, [install +Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### AWS CLI + +If you do not have the AWS CLI installed, install. For details, see the [AWS +documentation](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html). + +### kubectl + +If you do not have `kubectl`, install. See the [Amazon EKS: install `kubectl` +documentation](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) +for details. + +### Helm 3.2.0+ + +If you do not have Helm 3.2.0+, install. For details, see the [Helm +documentation](https://helm.sh/docs/intro/install/). + +## Procedure + +1. Open a Terminal window. + +1. Configure AWS CLI with your AWS credentials. For details, see the [AWS + documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). + +1. Go to the Terraform directory for your Materialize deployment. For example, + if you deployed from the `examples/simple` directory: + + ```bash + cd terraform-aws-materialize/examples/simple + ``` + +1. Optional. You may need to update your fork of the Terraform module to + upgrade. + + {{< tip >}} + {{% self-managed/aws-terraform-upgrade-notes %}} + + See [Materialize on AWS releases](/self-managed/installation/appendix-terraforms/#materialize-on-aws-terraform-module) for notable changes. + + {{}} + + +1. Configure `kubectl` to connect to your EKS cluster, replacing: + + - `` with the name of your EKS cluster. Your cluster + name has the form `{namespace}-{environment}-eks`; e.g., + `my-demo-dev-eks`. + + - `` with the region of your EKS cluster. The + simple example uses `us-east-1`. + + ```bash + aws eks update-kubeconfig --name --region + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl get nodes + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +{{% self-managed/versions/upgrade/upgrade-steps-cloud %}} diff --git a/doc/user/content/self-managed/installation/install-on-azure/_index.md b/doc/user/content/self-managed/installation/install-on-azure/_index.md new file mode 100644 index 0000000000000..9a07ffc600377 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-azure/_index.md @@ -0,0 +1,517 @@ +--- +title: "Install on Azure (via Terraform)" +description: "Install Materialize on Azure Kubernetes Service (AKS) using Terraform." +disable_list: true +menu: + main: + parent: "installation" + identifier: "install-on-azure" + weight: 30 +--- + +{{% self-managed/materialize-components-sentence blobstorage="blob storage; specifically **block** blob storage on Azure" %}} + +The tutorial deploys Materialize to Azure Kubernetes Service (AKS) with a +PostgreSQL database as the metadata database and Azure premium block blob +storage for blob storage. The tutorial uses [Materialize on Azure Terraform +modules](https://github.com/MaterializeInc/terraform-azurerm-materialize) to: + +- Set up the Azure Kubernetes environment +- Call + [terraform-helm-materialize](https://github.com/MaterializeInc/terraform-helm-materialize) + module to deploy Materialize Operator and Materialize instances to that AKS + cluster + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< self-managed/tutorial-disclaimer >}} + +{{< /warning >}} + +## Prerequisites + +### Azure subscription + +If you do not have an Azure subscription to use for this tutorial, create one. + +### Azure CLI + +If you don't have Azure CLI installed, [install Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli). + +### Terraform + +If you don't have Terraform installed, [install Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### kubectl + +If you do not have `kubectl`, install `kubectl`. + +### Python (v3.12+) and pip + +If you don't have Python (v3.12 or greater) installed, install it. See +[Python.org](https://www.python.org/downloads/). If `pip` is not included with +your version of Python, install it. + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see to +the [Helm documentation](https://helm.sh/docs/intro/install/). + +### jq (Optional) + +*Optional*. `jq` is used to parse the AKS cluster name and region from the +Terraform outputs. Alternatively, you can manually specify the name and region. +If you want to use `jq` and do not have `jq` installed, install. + +## A. Authenticate with Azure + +1. Open a Terminal window. + +1. Authenticate with Azure. + + ```bash + az login + ``` + + The command opens a browser window to sign in to Azure. Sign in. + +1. Select the subscription and tenant to use. After you have signed in, back in + the terminal, your tenant and subscription information is displayed. + + ```none + Retrieving tenants and subscriptions for the selection... + + [Tenant and subscription selection] + + No Subscription name Subscription ID Tenant + ----- ------------------- ------------------------------------ ---------------- + [1]* ... ... ... + + The default is marked with an *; the default tenant is '' and + subscription is '' (). + ``` + + Select the subscription and tenant. + +1. Set `ARM_SUBSCRIPTION_ID` to the subscription ID. + + ```bash + export ARM_SUBSCRIPTION_ID= + ``` + +## B. Set up Azure Kubernetes environment and install Materialize + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< /warning >}} + +{{< tabs >}} + +{{< tab "Deployed components" >}} + +[Materialize on Azure Terraform +module](https://github.com/MaterializeInc/terraform-azurerm-materialize) for +deploys a sample infrastructure on Azure with the following components: + +{{< yaml-table data="self_managed/azure_terraform_deployed_components" >}} + +{{< tip >}} + +{{% self-managed/azure-terraform-configs %}} + +{{< /tip >}} + +{{}} +{{< tab "Releases" >}} + +{{< yaml-table data="self_managed/azure_terraform_versions" >}} + +{{}} +{{}} + +1. Open a Terminal window. + +{{% self-managed/versions/step-clone-azure-terraform-repo %}} + +1. Go to the `examples/simple` folder in the Materialize Terraform repo + directory. + + ```bash + cd terraform-azurerm-materialize/examples/simple + ``` + + {{< tip >}} + + {{% self-managed/azure-terraform-configs %}} + + {{< /tip >}} + + +1. Optional. Create a virtual environment, specifying a path for the new virtual + environment: + + ```bash + python3 -m venv + + ``` + + Activate the virtual environment: + ```bash + source /bin/activate + ``` + +1. Install the required packages. + + ```bash + pip install -r requirements.txt + ``` + +1. Create a `terraform.tfvars` file (you can copy from the + `terraform.tfvars.example` file) and specify: + + - The prefix for the resources. Prefix has a maximum of 12 characters and + contains only alphanumeric characters and hyphens; e.g., `mydemo`. + + - The location for the AKS cluster. + + ```bash + prefix="enter-prefix" // maximum 12 characters, containing only alphanumeric characters and hyphens; e.g. mydemo + location="eastus2" + ``` + + {{< tip >}} + + {{% self-managed/azure-terraform-configs %}} + + {{< /tip >}} + +1. Initialize the terraform directory. + + ```bash + terraform init + ``` + +1. Use terraform plan to review the changes to be made. + + ```bash + terraform plan + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply + ``` + + To approve the changes and apply, enter `yes`. + + Upon successful completion, various fields and their values are output: + + ```bash + Apply complete! Resources: 33 added, 0 changed, 0 destroyed. + + Outputs: + + aks_cluster = + connection_strings = + kube_config = + load_balancer_details = {} + resource_group_name = "mydemo-rg" + ``` + +1. Configure `kubectl` to connect to your cluster: + + - ``. Your cluster name has the form `-aks`; e.g., + `mz-simple-aks`. + + - ``, as specified in the output. + + ```bash + az aks get-credentials --resource-group --name + ``` + + Alternatively, you can use the following command to get the cluster name and + resource group name from the Terraform output: + + ```bash + az aks get-credentials --resource-group $(terraform output -raw resource_group_name) --name $(terraform output -json aks_cluster | jq -r '.name') + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl cluster-info + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +1. By default, the example Terraform installs the Materialize Operator and, + starting in v0.3.0, a `cert-manager`. Verify the + installation and check the status: + + {{< tabs >}} + {{< tab "Materialize Operator" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/materialize-mydemo-materialize-operator-74d8f549d6-lkjjf 1/1 Running 0 36m + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/materialize-mydemo-materialize-operator 1/1 1 1 36m + + NAME DESIRED CURRENT READY AGE + replicaset.apps/materialize-mydemo-materialize-operator-74d8f549d6 1 1 1 36m + ``` + + {{}} + + {{< tab "cert-manager (Starting in version 0.3.0)" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n cert-manager + ``` + Wait for the components to be in the `Running` state: + ``` + NAME READY STATUS RESTARTS AGE + pod/cert-manager-8576d99cc8-xqxbc 1/1 Running 0 4m22s + pod/cert-manager-cainjector-664b5878d6-wc4tz 1/1 Running 0 4m22s + pod/cert-manager-webhook-6ddb7bd6c5-vrm2p 1/1 Running 0 4m22s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/cert-manager ClusterIP 10.1.227.230 9402/TCP 4m22s + service/cert-manager-cainjector ClusterIP 10.1.222.156 9402/TCP 4m22s + service/cert-manager-webhook ClusterIP 10.1.84.207 443/TCP,9402/TCP 4m22s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/cert-manager 1/1 1 1 4m23s + deployment.apps/cert-manager-cainjector 1/1 1 1 4m23s + deployment.apps/cert-manager-webhook 1/1 1 1 4m23s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/cert-manager-8576d99cc8 1 1 1 4m23s + replicaset.apps/cert-manager-cainjector-664b5878d6 1 1 1 4m23s + replicaset.apps/cert-manager-webhook-6ddb7bd6c5 1 1 1 4m23s + ``` + + {{}} + {{}} + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting/). + +1. Once the Materialize operator is deployed and running, you can deploy the + Materialize instances. To deploy Materialize instances, create a + `mz_instances.tfvars` file with the Materialize instance configuration. + + For example, the following specifies the configuration for a `demo` instance. + + ```bash + cat < mz_instances.tfvars + + materialize_instances = [ + { + name = "demo" + namespace = "materialize-environment" + database_name = "demo_db" + cpu_request = "1" + memory_request = "2Gi" + memory_limit = "2Gi" + } + ] + EOF + ``` + + - **Starting in v0.3.0**, the Materialize on Azure Terraform module also + deploys, by default, a self-signed `ClusterIssuer`. The `ClusterIssuer` is + deployed after the `cert-manager` is deployed and running. + + - **Starting in v0.3.1**, the Materialize on Azure Terraform module also + deploys, by default, [Load + balancers](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_materialize_instances) + for Materialize instances (i.e., the + [`create_load_balancer`](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_materialize_instances) + flag defaults to `true`). The load balancers, by default, are configured to + be internal (i.e., the + [`internal_load_balancer`](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_materialize_instances) + flag defaults to `true`). + + - **Starting in v0.4.3**, you can specify addition configuration options via + `environmentd_extra_args`. + + {{< tip >}} + {{% self-managed/azure-terraform-upgrade-notes %}} + + See [Materialize on Azure releases](/self-managed/installation/appendix-terraforms/#materialize-on-azure-terraform-module) for notable changes. + {{}} + +1. Run `terraform plan` with both `.tfvars` files and review the changes to be + made. + + ```bash + terraform plan -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + The plan should show the changes to be made, with a summary similar to the + following: + + ``` + Plan: 9 to add, 1 to change, 0 to destroy. + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + To approve the changes and apply, enter `yes`. + + + + Upon successful completion, you should see output with a summary similar to the following: + + ```bash + Apply complete! Resources: 9 added, 1 changed, 0 destroyed. + + Outputs: + + aks_cluster = + connection_strings = + kube_config = + load_balancer_details = { + "demo" = { + "balancerd_load_balancer_ip" = "192.0.2.10" + "console_load_balancer_ip" = "192.0.2.254" + } + } + resource_group_name = "mydemo-rg" + ``` + +1. Verify the installation and check the status: + + ```bash + kubectl get all -n materialize-environment + ``` + + Wait for the components to be ready and in the `Running` state. + + ```none + NAME READY STATUS RESTARTS AGE + pod/db-demo-db-l6ss8 0/1 Completed 0 2m21s + pod/mz62lr3yltj8-balancerd-6d5dd6d4cf-r9nf4 1/1 Running 0 111s + pod/mz62lr3yltj8-cluster-s2-replica-s1-gen-1-0 1/1 Running 0 114s + pod/mz62lr3yltj8-cluster-u1-replica-u1-gen-1-0 1/1 Running 0 114s + pod/mz62lr3yltj8-console-bfc797745-6nlwv 1/1 Running 0 96s + pod/mz62lr3yltj8-console-bfc797745-tk9vm 1/1 Running 0 96s + pod/mz62lr3yltj8-environmentd-1-0 1/1 Running 0 2m4s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/mz62lr3yltj8-balancerd ClusterIP None 6876/TCP,6875/TCP 111s + service/mz62lr3yltj8-balancerd-lb LoadBalancer 10.1.201.77 192.0.2.10 6875:30890/TCP,6876:31750/TCP 2m4s + service/mz62lr3yltj8-cluster-s2-replica-s1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 114s + service/mz62lr3yltj8-cluster-u1-replica-u1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 114s + service/mz62lr3yltj8-console ClusterIP None 8080/TCP 96s + service/mz62lr3yltj8-console-lb LoadBalancer 10.1.130.212 192.0.2.254 8080:30379/TCP 2m4s + service/mz62lr3yltj8-environmentd ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 111s + service/mz62lr3yltj8-environmentd-1 ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 2m5s + service/mz62lr3yltj8-persist-pubsub-1 ClusterIP None 6879/TCP 2m4s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/mz62lr3yltj8-balancerd 1/1 1 1 111s + deployment.apps/mz62lr3yltj8-console 2/2 2 2 96s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/mz62lr3yltj8-balancerd-6d5dd6d4cf 1 1 1 111s + replicaset.apps/mz62lr3yltj8-console-bfc797745 2 2 2 96s + + NAME READY AGE + statefulset.apps/mz62lr3yltj8-cluster-s2-replica-s1-gen-1 1/1 114s + statefulset.apps/mz62lr3yltj8-cluster-u1-replica-u1-gen-1 1/1 114s + statefulset.apps/mz62lr3yltj8-environmentd-1 1/1 2m4s + + NAME STATUS COMPLETIONS DURATION AGE + job.batch/db-demo-db Complete 1/1 10s 2m21s + + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting/). + +1. Open the Materialize Console in your browser: + + + {{< tabs >}} + + {{< tab "Via Network Load Balancer" >}} + + Starting in v0.3.1, for each Materialize instance, Materialize on Azure + Terraform module also deploys load balancers (by default, internal) with the + following listeners, including a listener on port 8080 for the Materialize + Console: + + | Port | Description | + | ---- | ------------| + | 6875 | For SQL connections to the database | + | 6876 | For HTTP(S) connections to the database | + | **8080** | **For HTTP(S) connections to Materialize Console** | + + The load balancer details are found in the `load_balancer_details` in + the [Terraform output](#azure-terraform-output). + + The example uses a self-signed ClusterIssuer. As such, you may encounter a + warning with regards to the certificate. In production, run with certificates + from an official Certificate Authority (CA) rather than self-signed + certificates. + + {{}} + + {{< tab "Via port forwarding" >}} + + {{% self-managed/port-forwarding-handling %}} + + {{}} + {{}} + + {{< tip >}} + + {{% self-managed/troubleshoot-console-mz_catalog_server_blurb %}} + + {{< /tip >}} + +## Next steps + +{{% self-managed/next-steps %}} + +## Cleanup + +{{% self-managed/cleanup-cloud %}} + + {{< tip>}} + + If the `terraform destroy` command is unable to delete the subnet because it + is in use, you can rerun the `terraform destroy` command. + + {{}} + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Appendix: Azure deployment guidelines](/self-managed/installation/install-on-azure/ + appendix-deployment-guidelines) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-azure/appendix-azure-configuration.md b/doc/user/content/self-managed/installation/install-on-azure/appendix-azure-configuration.md new file mode 100644 index 0000000000000..3a3eedaacc7a5 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-azure/appendix-azure-configuration.md @@ -0,0 +1,85 @@ +--- +title: "Appendix: Required configuration" +description: "Required configuration for Materialize on Azure Terraform." +menu: + main: + parent: "install-on-azure" + identifier: "appendix-azure-config" + weight: 50 +--- + +When using the root `main.tf` file from the [Materialize on Azure Terraform +module](https://github.com/MaterializeInc/terraform-azurerm-materialize), the +following configurations are required. [^1] + +## Required variables + +When using the root `main.tf` file from the [Materialize on Azure Terraform +module](https://github.com/MaterializeInc/terraform-azurerm-materialize), the +following variables must be set: [^1] + +{{< yaml-table data="self_managed/azure_required_variables" >}} + +For a list of all variables, see the +[README.md](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#inputs) +or the [`variables.tf` file](https://github.com/MaterializeInc/terraform-azurerm-materialize/blob/main/variables.tf). + +## Resource group + +When using the root `main.tf` file from the [Materialize on Azure Terraform +module](https://github.com/MaterializeInc/terraform-azurerm-materialize), an +Azure Resource Group `azurerm_resource_group` is required.[^1] You can either +create a new resource group or use an existing resource group: + +- **To create a new resource group**, declare the resource group to create in + your configuration: + + ```hcl + resource "azurerm_resource_group" "materialize" { + name = var.resource_group_name + location = var.location # Defaults to eastus2 + tags = var.tags # Optional + } + ``` + +- **To use an existing resource group**, set the [`resource_group_name` + variable](https://github.com/MaterializeInc/terraform-azurerm-materialize/blob/main/variables.tf) + to that group's name. + +## Required providers + +When using the root `main.tf` file from the [Materialize on Azure Terraform +module +v0.2.0+](https://github.com/MaterializeInc/terraform-azurerm-materialize), the +following provider declarations are required: [^1] + +```hcl +provider "azurerm" { + # Set the Azure subscription ID here or use the ARM_SUBSCRIPTION_ID environment variable + # subscription_id = "XXXXXXXXXXXXXXXXXXX" + + # Specify addition Azure provider configuration as needed + + features { } +} + + +provider "kubernetes" { + host = module.aks.cluster_endpoint + client_certificate = base64decode(module.aks.kube_config[0].client_certificate) + client_key = base64decode(module.aks.kube_config[0].client_key) + cluster_ca_certificate = base64decode(module.aks.kube_config[0].cluster_ca_certificate) +} + +provider "helm" { + kubernetes { + host = module.aks.cluster_endpoint + client_certificate = base64decode(module.aks.kube_config[0].client_certificate) + client_key = base64decode(module.aks.kube_config[0].client_key) + cluster_ca_certificate = base64decode(module.aks.kube_config[0].cluster_ca_certificate) + } +} +``` + +[^1]: If using the `examples/simple/main.tf`, the example configuration handles +them for you. diff --git a/doc/user/content/self-managed/installation/install-on-azure/appendix-deployment-guidelines.md b/doc/user/content/self-managed/installation/install-on-azure/appendix-deployment-guidelines.md new file mode 100644 index 0000000000000..731040008432b --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-azure/appendix-deployment-guidelines.md @@ -0,0 +1,101 @@ +--- +title: "Appendix: Azure deployment guidelines" +description: "Azure environment setup/deployment guidelines" +menu: + main: + parent: "install-on-azure" + identifier: "azure-deployment-guidelines" + weight: 40 +--- + +## Recommended instance types + +As a general guideline, we recommend: + +- Processor Type: ARM-based CPU + +- Sizing: 2:1 disk-to-RAM ratio with spill-to-disk enabled. + +### Recommended Azure VM Types with Local NVMe Disks + +When operating on Azure in production, we recommend [Epdsv6 +sizes](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/memory-optimized/epdsv6-series?tabs=sizebasic#sizes-in-series) +Azure VM Types with Local NVMe Disk: + +| VM Size | vCPUs | Memory | Ephemeral Disk | Disk-to-RAM Ratio | +| ------------------ | ----- | ------- | -------------- | ----------------- | +| Standard_E2pds_v6 | 2 | 16 GiB | 75 GiB | ~4.7:1 | +| Standard_E4pds_v6 | 4 | 32 GiB | 150 GiB | ~4.7:1 | +| Standard_E8pds_v6 | 8 | 64 GiB | 300 GiB | ~4.7:1 | +| Standard_E16pds_v6 | 16 | 128 GiB | 600 GiB | ~4.7:1 | +| Standard_E32pds_v6 | 32 | 256 GiB | 1,200 GiB | ~4.7:1 | + +{{< important >}} + +These VM types provide **ephemeral** local NVMe SSD disks. Data is lost +when the VM is stopped or deleted. + +{{}} + +See also [Locally attached NVMe storage](#locally-attached-nvme-storage). + +## Locally-attached NVMe storage + +For optimal performance, Materialize requires fast, locally-attached NVMe +storage. Having a locally-attached storage allows Materialize to spill to disk +when operating on datasets larger than main memory as well as allows for a more +graceful degradation rather than OOMing. Network-attached storage (like EBS +volumes) can significantly degrade performance and is not supported. + +Starting in v0.4.0 of Materialize on Azure Terraform, disk support (using +OpenEBS and NVMe instance storage) is enabled, by default, for Materialize. With +this change, the Terraform: + +- Installs OpenEBS via Helm; + +- Configures NVMe instance store volumes using a bootstrap script; + +- Creates appropriate storage classes for Materialize. + +Associated with this change: + +- The following configuration options are available: + + - [`enable_disk_support`] + - [`disk_support_config`] + - [`disk_setup_image`] + +- The default +[`aks_config.vm_size`](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_aks_config) +has changed from `Standard_E8ps_v6` to `Standard_E4pds_v6`. See [Recommended +instance types](#recommended-instance-types). + +## Recommended Azure Blob Storage + +Materialize writes **block** blobs on Azure. As a general guideline, we +recommend **Premium block blob** storage accounts. + +## CPU affinity + +It is strongly recommended to enable the Kubernetes `static` [CPU management policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy). +This ensures that each worker thread of Materialize is given exclusively access to a vCPU. Our benchmarks have shown this +to substantially improve the performance of compute-bound workloads. + +## TLS + +When running with TLS in production, run with certificates from an official +Certificate Authority (CA) rather than self-signed certificates. + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) + +[`enable_disk_support`]: https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#disk-support-for-materialize-on-azure + +[`disk_support_config`]: + https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_disk_support_config + +[`disk_setup_image`]: + https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_disk_setup_image diff --git a/doc/user/content/self-managed/installation/install-on-azure/upgrade-on-azure.md b/doc/user/content/self-managed/installation/install-on-azure/upgrade-on-azure.md new file mode 100644 index 0000000000000..2927a124bafd4 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-azure/upgrade-on-azure.md @@ -0,0 +1,193 @@ +--- +title: "Upgrade on Azure" +description: "Procedure to upgrade your Materialize operator and instances running on Azure" +menu: + main: + parent: "install-on-azure" + identifier: "upgrade-on-azure" + weight: 10 +--- + +To upgrade your Materialize instances, upgrade the Materialize operator first +and then the Materialize instances. The following tutorial upgrades your +Materialize deployment running on Azure Kubernetes Service (AKS) + +The tutorial assumes you have installed Materialize on Azure Kubernetes Service +(AKS) using the instructions on [Install on +Azure](/self-managed/installation/install-on-azure/) (either from the examples/simple +directory or the root). + +## Version compatibility + +{{< tabs >}} + +{{< tab "Helm chart releases" >}} + +The following table presents the versions compatibility for the operator and the +applications: + +{{< yaml-table data="self_managed/self_managed_operator_compatibility" >}} + +{{}} + +{{< tab "Materialize on Azure Terraform Releases" >}} + +When upgrading, you may need or want to update your fork of the Terraform module +to upgrade. + +{{< yaml-table data="self_managed/azure_terraform_versions" >}} + +{{}} +{{}} + +## Prerequisites + +{{< important >}} + +The following procedure performs an in-place upgrade, which incurs downtime. + +To perform a rolling upgrade(where both the old and new Materialize instances +are running before the the old instances are removed), you can specify +`inPlaceRollout` to false. When performing a rolling upgrade, ensure you have +enough resources to support having both the old and new Materialize instances +running. + +{{}} + +### Azure subscription + +If you do not have an Azure subscription to use for this tutorial, create one. + +### Azure CLI + +If you don't have Azure CLI installed, [install Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli). + +### Terraform + +If you don't have Terraform installed, [install Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### kubectl + +If you do not have `kubectl`, install `kubectl`. + +### Python (v3.12+) and pip + +If you don't have Python (v3.12 or greater) installed, install it. See +[Python.org](https://www.python.org/downloads/). If `pip` is not included with +your version of Python, install it. + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see to +the [Helm documentation](https://helm.sh/docs/intro/install/). + +### jq (Optional) + +*Optional*. `jq` is used to parse the AKS cluster name and region from the +Terraform outputs. Alternatively, you can manually specify the name and region. +If you want to use `jq` and do not have `jq` installed, install. + +## A. Authenticate with Azure + +1. Open a Terminal window. + +1. Authenticate with Azure. + + ```bash + az login + ``` + + The command opens a browser window to sign in to Azure. Sign in. + +1. Select the subscription and tenant to use. After you have signed in, back in + the terminal, your tenant and subscription information is displayed. + + ```none + Retrieving tenants and subscriptions for the selection... + + [Tenant and subscription selection] + + No Subscription name Subscription ID Tenant + ----- ------------------- ------------------------------------ ---------------- + [1]* ... ... ... + + The default is marked with an *; the default tenant is '' and + subscription is '' (). + ``` + + Select the subscription and tenant. + +1. Set `ARM_SUBSCRIPTION_ID` to the subscription ID. + + ```bash + export ARM_SUBSCRIPTION_ID= + ``` + +## B. Upgrade process + +1. Go to the Terraform directory for your Materialize deployment. For example, + if you deployed from the `examples/simple` directory: + + ```bash + cd terraform-azurerm-materialize/examples/simple + ``` + +1. Optional. You may need to update your fork of the Terraform module to + upgrade. + + {{< yaml-table data="self_managed/azure_terraform_versions" >}} + + {{< tip >}} + + {{% self-managed/azure-terraform-upgrade-notes %}} + + {{}} + +1. Optional. Create a virtual environment, specifying a path for the new virtual + environment: + + ```bash + python3 -m venv + ``` + + Activate the virtual environment: + ```bash + source /bin/activate + ``` + +1. Install the required packages. + + ```bash + pip install -r requirements.txt + ``` + +1. Configure `kubectl` to connect to your cluster: + + - ``. Your cluster name has the form `-aks`; e.g., + `mz-simple-aks`. + + - ``, as specified in the output. You resource group + name has the form `-rg`; e.g., `mz-simple-rg`. + + ```bash + az aks get-credentials --resource-group --name + ``` + + Alternatively, you can use the following command to get the cluster name and + resource group name from the Terraform output (in your `terraform.tfstate` + file): + + ```bash + az aks get-credentials --resource-group $(terraform output -raw resource_group_name) --name $(terraform output -json aks_cluster | jq -r '.name') + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl get nodes + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +{{% self-managed/versions/upgrade/upgrade-steps-cloud %}} diff --git a/doc/user/content/self-managed/installation/install-on-gcp/_index.md b/doc/user/content/self-managed/installation/install-on-gcp/_index.md new file mode 100644 index 0000000000000..9278f1fdc8a64 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-gcp/_index.md @@ -0,0 +1,578 @@ +--- +title: "Install on GCP (via Terraform)" +description: "Install Materialize on GCP Google Kubernetes Engine (GKE) using Terraform." +aliases: + - /self-hosted/install-on-gcp/ +disable_list: true +menu: + main: + parent: "installation" + identifier: "install-on-gcp" + weight: 40 +--- + +{{% self-managed/materialize-components-sentence %}} + +This tutorial deploys Materialize to GCP Google Kubernetes Engine (GKE) cluster +with a Cloud SQL PostgreSQL database as the metadata database and Cloud Storage +bucket for blob storage. Specifically, the tutorial uses [Materialize on Google +Cloud Provider Terraform +module](https://github.com/MaterializeInc/terraform-google-materialize) to: + +- Set up the GCP environment. + +- Call + [terraform-helm-materialize](https://github.com/MaterializeInc/terraform-helm-materialize) + module to deploy Materialize Operator and Materialize instances to the GKE + cluster. + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< self-managed/tutorial-disclaimer >}} + +{{< /warning >}} + +## Prerequisites + +### Google cloud provider project + +You need a GCP project for which you have a role (such as +`roles/resourcemanager.projectIamAdmin` or `roles/owner`) that includes [ +permissions to manage access to the +project](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + +### gcloud CLI + +If you do not have gcloud CLI, install. For details, see the [Install the gcloud +CLI documentation](https://cloud.google.com/sdk/docs/install). + +### Google service account + +The tutorial assumes the use of a service account. If you do not have a service +account to use for this tutorial, create a service account. For details, see +[Create service +accounts](https://cloud.google.com/iam/docs/service-accounts-create#creating). + +### Terraform + +If you do not have Terraform installed, [install +Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### kubectl and plugins + +{{< tip >}} + +Using `gcloud` to install `kubectl` will also install the needed plugins. +Otherwise, you will need to manually install the `gke-gcloud-auth-plugin` for +`kubectl`. + +{{< /tip >}} + +- If you do not have `kubectl`, install `kubectl`. To install, see [Install + kubectl and configure cluster + access](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) + for details. You will configure `kubectl` to interact with your GKE cluster + later in the tutorial. + +- If you do not have `gke-gcloud-auth-plugin` for `kubectl`, install the + `gke-gcloud-auth-plugin`. For details, see [Install the + gke-gcloud-auth-plugin](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_plugin). + +### Helm 3.2.0+ + +If you do not have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### jq (Optional) + +*Optional*. `jq` is used to parse the GKE cluster name and region from the +Terraform outputs. Alternatively, you can manually specify the name and region. +If you want to use `jq` and do not have `jq` installed, install. + +## A. Configure GCP project and service account + +1. Open a Terminal window. + +1. Initialize the gcloud CLI (`gcloud init`) to specify the GCP project you want + to use. For details, see the [Initializing the gcloud CLI + documentation](https://cloud.google.com/sdk/docs/initializing#initialize_the). + + {{< tip >}} + You do not need to configure a default Compute Region and Zone as you will + specify the region. + {{}} + +1. Enable the following services for your GCP project, if not already enabled: + + ```bash + gcloud services enable container.googleapis.com # For creating Kubernetes clusters + gcloud services enable sqladmin.googleapis.com # For creating databases + gcloud services enable cloudresourcemanager.googleapis.com # For managing GCP resources + gcloud services enable servicenetworking.googleapis.com # For private network connections + gcloud services enable iamcredentials.googleapis.com # For security and authentication + ``` + +1. To the service account that will run the Terraform script, + grant the following IAM roles: + + - `roles/editor` + - `roles/iam.serviceAccountAdmin` + - `roles/servicenetworking.networksAdmin` + - `roles/storage.admin` + - `roles/container.admin` + + 1. Enter your GCP project ID. + + ```bash + read -s PROJECT_ID + ``` + + 1. Find your service account email for your GCP project + + ```bash + gcloud iam service-accounts list --project $PROJECT_ID + ``` + + 1. Enter your service account email. + + ```bash + read -s SERVICE_ACCOUNT + ``` + + 1. Grant the service account the neccessary IAM roles. + + ```bash + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/editor" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/iam.serviceAccountAdmin" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/servicenetworking.networksAdmin" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/storage.admin" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/container.admin" + ``` + +1. For the service account, authenticate to allow Terraform to + interact with your GCP project. For details, see [Terraform: Google Cloud + Provider Configuration + reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#authentication). + + For example, if using [User Application Default + Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default), + you can run the following command: + + ```bash + gcloud auth application-default login + ``` + + {{< tip >}} + If using `GOOGLE_APPLICATION_CREDENTIALS`, use absolute path to your key file. + {{}} + +## B. Set up GCP Kubernetes environment and install Materialize + +{{< warning >}} + +{{< self-managed/terraform-disclaimer >}} + +{{< /warning >}} + +{{< tabs >}} + +{{< tab "Deployed components" >}} +[Materialize on GCP Terraform +module](https://github.com/MaterializeInc/terraform-google-materialize) deploys +a sample infrastructure on GCP (region `us-central1`) with the following +components: + +{{< yaml-table data="self_managed/gcp_terraform_deployed_components" >}} + +{{< tip >}} +{{< self-managed/gcp-terraform-configs >}} +{{< /tip >}} +{{}} +{{< tab "Releases" >}} + +{{< yaml-table data="self_managed/gcp_terraform_versions" >}} + +{{}} +{{}} + +{{% self-managed/versions/step-clone-google-terraform-repo %}} + +1. Go to the `examples/simple` folder in the Materialize Terraform repo + directory. + + ```bash + cd terraform-google-materialize/examples/simple + ``` + + {{< tip >}} + {{< self-managed/gcp-terraform-configs >}} + {{< /tip >}} + +1. Create a `terraform.tfvars` file (you can copy from the + `terraform.tfvars.example` file) and specify the following variables: + + | **Variable** | **Description** | + |--------------|-----------------| + | `project_id` | Your GCP project ID. | + | `prefix` | A prefix (e.g., `mz-simple`) for your resources. Prefix has a maximum of 15 characters and contains only alphanumeric characters and dashes. | + | `region` | The region for the GKE cluster. | + + ```bash + project_id = "enter-your-gcp-project-id" + prefix = "enter-your-prefix" // Maximum of 15 characters, contain lowercase alphanumeric and hyphens only (e.g., mz-simple) + region = "us-central1" + ``` + + {{< tip >}} + + {{< self-managed/gcp-terraform-configs >}} + + {{< /tip >}} + +1. Initialize the terraform directory. + + ```bash + terraform init + ``` + +1. Run terraform plan and review the changes to be made. + + ```bash + terraform plan + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply + ``` + + To approve the changes and apply, enter `yes`. + + Upon successful completion, various fields and their values are output: + + ```bash + Apply complete! Resources: 27 added, 0 changed, 0 destroyed. + + Outputs: + + connection_strings = + gke_cluster = + load_balancer_details = {} + network = { + "network_id" = "projects/my-project/global/networks/mz-simple-network" + "network_name" = "mz-simple-network" + "subnet_name" = "mz-simple-subnet" + } + service_accounts = { + "gke_sa" = "mz-simple-gke-sa@my-project.iam.gserviceaccount.com" + "materialize_sa" = "mz-simple-materialize-sa@my-project.iam.gserviceaccount.com" + } + ``` + +1. Configure `kubectl` to connect to your GKE cluster, specifying: + + - ``. Your cluster name has the form `-gke`; e.g., + `mz-simple-gke`. + + - ``. By default, the example Terraform module uses the `us-central1` + region. + + - ``. Your GCP project ID. + + ```bash + gcloud container clusters get-credentials \ + --region \ + --project + ``` + + Alternatively, you can use the following command to get the cluster name and + region from the Terraform output and the project ID from the environment + variable set earlier. + + ```bash + gcloud container clusters get-credentials $(terraform output -json gke_cluster | jq -r .name) \ + --region $(terraform output -json gke_cluster | jq -r .location) --project $PROJECT_ID + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl cluster-info + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +1. By default, the example Terraform installs the Materialize Operator and, + starting in v0.3.0, a `cert-manager`. Verify the + installation and check the status: + + {{< tabs >}} + {{< tab "Materialize Operator" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/materialize-mz-simple-materialize-operator-74d8f549d6-lkjjf 1/1 Running 0 36m + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/materialize-mz-simple-materialize-operator 1/1 1 1 36m + + NAME DESIRED CURRENT READY AGE + replicaset.apps/materialize-mz-simple-materialize-operator-74d8f549d6 1 1 1 36m + ``` + + {{}} + {{< tab "cert-manager (Starting in version 0.3.0)" >}} + + Verify the installation and check the status: + + ```shell + kubectl get all -n cert-manager + ``` + Wait for the components to be in the `Running` state: + ``` + NAME READY STATUS RESTARTS AGE + pod/cert-manager-6794b8d569-vt264 1/1 Running 0 22m + pod/cert-manager-cainjector-7f69cd69f7-7brqw 1/1 Running 0 22m + pod/cert-manager-webhook-6cc5dccc4b-7tmd4 1/1 Running 0 22m + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/cert-manager ClusterIP 10.52.3.63 9402/TCP 22m + service/cert-manager-cainjector ClusterIP 10.52.15.171 9402/TCP 22m + service/cert-manager-webhook ClusterIP 10.52.5.148 443/TCP,9402/TCP 22m + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/cert-manager 1/1 1 1 22m + deployment.apps/cert-manager-cainjector 1/1 1 1 22m + deployment.apps/cert-manager-webhook 1/1 1 1 22m + + NAME DESIRED CURRENT READY AGE + replicaset.apps/cert-manager-6794b8d569 1 1 1 22m + replicaset.apps/cert-manager-cainjector-7f69cd69f7 1 1 1 22m + replicaset.apps/cert-manager-webhook-6cc5dccc4b 1 1 1 22m + ``` + + {{}} + {{}} + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting/). + +1. Once the Materialize operator is deployed and running, you can deploy the + Materialize instances. To deploy Materialize instances, create a + `mz_instances.tfvars` file with the Materialize instance configuration. + + For example, the following specifies the configuration for a `demo` instance. + + ```bash + cat < mz_instances.tfvars + + materialize_instances = [ + { + name = "demo" + namespace = "materialize-environment" + database_name = "demo_db" + cpu_request = "1" + memory_request = "2Gi" + memory_limit = "2Gi" + } + ] + EOF + ``` + + - **Starting in v0.3.0**, the Materialize on GCP Terraform module also + deploys, by default: + + - [Load balancers](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_materialize_instances) for Materialize instances (i.e., the [`create_load_balancer`](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_materialize_instances) flag defaults to `true`). The load balancers, by default, are configured to be internal (i.e., the [`internal_load_balancer`](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_materialize_instances) flag defaults to `true`). + + - A self-signed `ClusterIssuer`. The `ClusterIssuer` is deployed after the + `cert-manager` is deployed and running. + + - **Starting in v0.4.3**, you can specify addition configuration options via + `environmentd_extra_args`. + + {{< tip >}} + {{% self-managed/gcp-terraform-upgrade-notes %}} + + See [Materialize on GCP releases](/self-managed/installation/appendix-terraforms/#materialize-on-gcp-terraform-module) for notable changes. + {{}} + +1. Run `terraform plan` with both `.tfvars` files and review the changes to be + made. + + ```bash + terraform plan -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + The plan should show the changes to be made, with a summary similar to the + following: + + ``` + Plan: 9 to add, 1 to change, 0 to destroy. + ``` + +1. If you are satisfied with the changes, apply. + + ```bash + terraform apply -var-file=terraform.tfvars -var-file=mz_instances.tfvars + ``` + + To approve the changes and apply, enter `yes`. + + + Upon successful completion, you should see output with a summary similar to the following: + + ```bash + Apply complete! Resources: 9 added, 1 changed, 0 destroyed. + + Outputs: + + connection_strings = + gke_cluster = + load_balancer_details = { + "demo" = { + "balancerd_load_balancer_ip" = "192.0.2.10" + "console_load_balancer_ip" = "192.0.2.254" + } + } + network = { + "network_id" = "projects/my-project/global/networks/mz-simple-network" + "network_name" = "mz-simple-network" + "subnet_name" = "mz-simple-subnet" + } + service_accounts = { + "gke_sa" = "mz-simple-gke-sa@my-project.iam.gserviceaccount.com" + "materialize_sa" = "mz-simple-materialize-sa@my-project.iam.gserviceaccount.com" + } + ``` + +1. Verify the installation and check the status: + + ```bash + kubectl get all -n materialize-environment + ``` + + Wait for the components to be in the `Running` state. + + ```none + NAME READY STATUS RESTARTS AGE + pod/db-demo-db-wrvhw 0/1 Completed 0 4m26s + pod/mzdtwvu4qe4q-balancerd-6989df5c75-mpmqx 1/1 Running 0 3m54s + pod/mzdtwvu4qe4q-cluster-s2-replica-s1-gen-1-0 1/1 Running 0 3m53s + pod/mzdtwvu4qe4q-cluster-u1-replica-u1-gen-1-0 1/1 Running 0 3m52s + pod/mzdtwvu4qe4q-console-7c9bc94bcb-6t7lg 1/1 Running 0 3m41s + pod/mzdtwvu4qe4q-console-7c9bc94bcb-9x5qq 1/1 Running 0 3m41s + pod/mzdtwvu4qe4q-environmentd-1-0 1/1 Running 0 4m9s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/mzdtwvu4qe4q-balancerd ClusterIP None 6876/TCP,6875/TCP 3m54s + service/mzdtwvu4qe4q-balancerd-lb LoadBalancer 10.52.5.105 192.0.2.10 6875:30844/TCP,6876:32307/TCP 4m9s + service/mzdtwvu4qe4q-cluster-s2-replica-s1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 3m53s + service/mzdtwvu4qe4q-cluster-u1-replica-u1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 3m52s + service/mzdtwvu4qe4q-console ClusterIP None 8080/TCP 3m41s + service/mzdtwvu4qe4q-console-lb LoadBalancer 10.52.4.2 192.0.2.254 8080:32193/TCP 4m9s + service/mzdtwvu4qe4q-environmentd ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 3m54s + service/mzdtwvu4qe4q-environmentd-1 ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 4m9s + service/mzdtwvu4qe4q-persist-pubsub-1 ClusterIP None 6879/TCP 4m9s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/mzdtwvu4qe4q-balancerd 1/1 1 1 3m54s + deployment.apps/mzdtwvu4qe4q-console 2/2 2 2 3m41s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/mzdtwvu4qe4q-balancerd-6989df5c75 1 1 1 3m54s + replicaset.apps/mzdtwvu4qe4q-console-7c9bc94bcb 2 2 2 3m41s + + NAME READY AGE + statefulset.apps/mzdtwvu4qe4q-cluster-s2-replica-s1-gen-1 1/1 3m53s + statefulset.apps/mzdtwvu4qe4q-cluster-u1-replica-u1-gen-1 1/1 3m52s + statefulset.apps/mzdtwvu4qe4q-environmentd-1 1/1 4m9s + + NAME STATUS COMPLETIONS DURATION AGE + job.batch/db-demo-db Complete 1/1 12s 4m27s + + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting/). + +1. Open the Materialize Console in your browser: + + {{< tabs >}} + + {{< tab "Via Network Load Balancer" >}} + + Starting in v0.3.0, for each Materialize instance, Materialize on GCP + Terraform module also deploys load balancers (by default, internal) with the + following listeners, including a listener on port 8080 for the Materialize + Console: + + | Port | Description | + | ---- | ------------| + | 6875 | For SQL connections to the database | + | 6876 | For HTTP(S) connections to the database | + | **8080** | **For HTTP(S) connections to Materialize Console** | + + The load balancer details are found in the `load_balancer_details` in + the [Terraform output](#gcp-terraform-output). + + The example uses a self-signed ClusterIssuer. As such, you may encounter a + warning with regards to the certificate. In production, run with certificates + from an official Certificate Authority (CA) rather than self-signed + certificates. + + {{}} + + {{< tab "Via port forwarding" >}} + + {{% self-managed/port-forwarding-handling console="console-lb"%}} + + {{}} + {{}} + + + {{< tip >}} + + {{% self-managed/troubleshoot-console-mz_catalog_server_blurb %}} + + {{< /tip >}} + +## Next steps + +{{% self-managed/next-steps %}} + +## Cleanup + +{{% self-managed/cleanup-cloud %}} + +## See also + +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Appendix: Google deployment guidelines](/self-managed/installation/install-on-gcp/appendix-deployment-guidelines/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-gcp/appendix-deployment-guidelines.md b/doc/user/content/self-managed/installation/install-on-gcp/appendix-deployment-guidelines.md new file mode 100644 index 0000000000000..b8d1ef54a5b79 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-gcp/appendix-deployment-guidelines.md @@ -0,0 +1,135 @@ +--- +title: "Appendix: GCP deployment guidelines" +description: "GCP environment setup/deployment guidelines" +menu: + main: + parent: "install-on-gcp" + identifier: "gcp-deployment-guidelines" + weight: 40 +--- + +## Recommended instance types + +As a general guideline, we recommend: + +- Processor Type: ARM-based CPU + +- Sizing: 2:1 disk-to-RAM ratio with spill-to-disk enabled. + +When operating on GCP in production, we recommend the following machine types +that support local SSD attachment: + +| Series | Examples | +| ------ | ---------- | +| [N2 high-memory series] | `n2-highmem-16` or `n2-highmem-32` with local NVMe SSDs | +| [N2D high-memory series] | `n2d-highmem-16` or `n2d-highmem-32` with local NVMe SSDs | + +To maintain the recommended 2:1 disk-to-RAM ratio for your machine type, see +[Number of local SSDs](#number-of-local-ssds) to determine the number of local +SSDs +([`disk_support_config.local_ssd_count`](https://github.com/MaterializeInc/terraform-google-materialize/blob/main/README.md#input_disk_support_config)) +to use. + +See also [Locally attached NVMe storage](#locally-attached-nvme-storage). + +## Number of local SSDs + +Each local NVMe SSD in GCP provides 375GB of storage. Use the appropriate number +of local SSDs +([`disk_support_config.local_ssd_count`](https://github.com/MaterializeInc/terraform-google-materialize/blob/main/README.md#input_disk_support_config)) +to ensure your total disk space is at least twice the amount of RAM in your +machine type for optimal Materialize performance. + +{{< note >}} + +Your machine type may only supports predefined number of local SSDs. For instance, `n2d-highmem-32` allows only the following number of local +SSDs: `4`,`8`,`16`, or `24`. To determine the valid number of Local SSDs to attach for your machine type, see the [GCP +documentation](https://cloud.google.com/compute/docs/disks/local-ssd#lssd_disk_options). + +{{}} + +For example, the following table provides a minimum local SSD count to ensure +the 2:1 disk-to-RAM ratio. Your actual +count will depend on the [your machine +type](https://cloud.google.com/compute/docs/disks/local-ssd#lssd_disk_options). + +| Machine Type | RAM | Required Disk | Minimum Local SSD Count | Total SSD Storage | +|-----------------|---------|---------------|-----------------------------|-------------------| +| `n2-highmem-8` | `64GB` | `128GB` | 1 | `375GB` | +| `n2-highmem-16` | `128GB` | `256GB` | 1 | `375GB` | +| `n2-highmem-32` | `256GB` | `512GB` | 2 | `750GB` | +| `n2-highmem-64` | `512GB` | `1024GB` | 3 | `1125GB` | +| `n2-highmem-80` | `640GB` | `1280GB` | 4 | `1500GB` | + +[N2 high-memory series]: https://cloud.google.com/compute/docs/general-purpose-machines#n2-high-mem + +[N2D high-memory series]: https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machine_types + +[enables spill-to-disk]: https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#disk-support-for-materialize-on-gcp + +## Locally-attached NVMe storage + +For optimal performance, Materialize requires fast, locally-attached NVMe +storage. Having a locally-attached storage allows Materialize to spill to disk +when operating on datasets larger than main memory as well as allows for a more +graceful degradation rather than OOMing. Network-attached storage (like EBS +volumes) can significantly degrade performance and is not supported. + +Starting in v0.4.0 of Materialize on Google Cloud Provider (GCP) Terraform, +disk support (using OpenEBS and NVMe instance storage) is enabled, by default, +for Materialize. With this change, the Terraform: + +- Installs OpenEBS via Helm; + +- Configures NVMe instance store volumes using a bootstrap script; + +- Creates appropriate storage classes for Materialize. + +Associated with this change: + +- The following configuration options are available: + + - [`enable_disk_support`] + - [`disk_support_config`] + +- The default [`gke_config.machine_type`] has changed from `e2-standard-4` to +`n2-highmem-8`. See [Recommended instance types](#recommended-instance-types). + +[enable disk support]: + https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#disk-support-for-materialize-on-gcp + +[`enable_disk_support`]: + https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_enable_disk_support + +[`disk_support_config`]: + https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_disk_support_config + +[`gke_config.machine_type`]: + https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_gke_config + +## CPU affinity + +It is strongly recommended to enable the Kubernetes `static` [CPU management policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy). +This ensures that each worker thread of Materialize is given exclusively access to a vCPU. Our benchmarks have shown this +to substantially improve the performance of compute-bound workloads. + +## TLS + +When running with TLS in production, run with certificates from an official +Certificate Authority (CA) rather than self-signed certificates. + +## Storage bucket versioning + +Starting in v0.3.1 of Materialize on GCP Terraform, storage bucket versioning is +disabled (i.e., +[`storage_bucket_versioning`](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_storage_bucket_versioning) +is set to `false` by default) to facilitate cleanup of resources during testing. +When running in production, versioning should be turned on with a sufficient TTL +([`storage_bucket_version_ttl`](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_storage_bucket_version_ttl)) +to meet any data-recovery requirements. + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/installation/install-on-gcp/appendix-gcp-configuration.md b/doc/user/content/self-managed/installation/install-on-gcp/appendix-gcp-configuration.md new file mode 100644 index 0000000000000..b8575b948a7d2 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-gcp/appendix-gcp-configuration.md @@ -0,0 +1,58 @@ +--- +title: "Appendix: Required configuration" +description: "Required configuration for Materialize on GCP Terraform." +menu: + main: + parent: "install-on-gcp" + identifier: "appendix-gcp-config" + weight: 50 +--- + +## Required variables + +The following variables are required when using the [Materialize on Google Cloud +Provider Terraform +module](https://github.com/MaterializeInc/terraform-google-materialize). + +{{< yaml-table data="self_managed/gcp_required_variables" >}} + +For a list of all variables, see the +[README.md](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#inputs) +or the [`variables.tf` file](https://github.com/MaterializeInc/terraform-google-materialize/blob/main/variables.tf). + +## Required providers and data source declaration + +To use [Materialize on Google Cloud Terraform +module](https://github.com/MaterializeInc/terraform-google-materialize) v0.2.0+, +you need to declare: + +- The following providers: + + ```hcl + provider "google" { + project = var.project_id + region = var.region + # Specify additional Google provider configuration as needed + } + + # Required for GKE authentication + provider "kubernetes" { + host = "https://${module.gke.cluster_endpoint}" + token = data.google_client_config.current.access_token + cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate) + } + + provider "helm" { + kubernetes { + host = "https://${module.gke.cluster_endpoint}" + token = data.google_client_config.current.access_token + cluster_ca_certificate = base64decode(module.gke.cluster_ca_certificate) + } + } + ``` + +- The following data source: + + ```hcl + data "google_client_config" "current" {} + ``` diff --git a/doc/user/content/self-managed/installation/install-on-gcp/upgrade-on-gcp.md b/doc/user/content/self-managed/installation/install-on-gcp/upgrade-on-gcp.md new file mode 100644 index 0000000000000..ecc99c4da21e2 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-gcp/upgrade-on-gcp.md @@ -0,0 +1,234 @@ +--- +title: "Upgrade on GCP" +description: "Procedure to upgrade your Materialize operator and instances running on GCP" +menu: + main: + parent: "install-on-gcp" + identifier: "upgrade-on-gcp" + weight: 10 +--- + +To upgrade your Materialize instances, upgrade the Materialize operator first +and then the Materialize instances. The following tutorial upgrades your +Materialize deployment running on GCP Google Kubernetes Engine (GKE). + +The tutorial assumes you have installed Materialize on GCP Google Kubernetes +Engine (GKE) using the instructions on [Install on +GCP](/self-managed/installation/install-on-gcp/) (either from the examples/simple directory +or the root). + +## Version compatibility + +{{< tabs >}} + +{{< tab "Helm chart releases" >}} + +The following table presents the versions compatibility for the operator and the +applications: + +{{< yaml-table data="self_managed/self_managed_operator_compatibility" >}} + +{{}} +{{< tab "Materialize on GCP Terraform Releases" >}} + +{{< yaml-table data="self_managed/gcp_terraform_versions" >}} + +{{}} +{{}} + +## Prerequisites + +{{< important >}} + +The following procedure performs an in-place upgrade, which incurs downtime. + +To perform a rolling upgrade(where both the old and new Materialize instances +are running before the the old instances are removed), you can specify +`inPlaceRollout` to false. When performing a rolling upgrade, ensure you have +enough resources to support having both the old and new Materialize instances +running. + +{{}} + +### Google cloud project + +You need a GCP project for which you have a role (such as +`roles/resourcemanager.projectIamAdmin` or `roles/owner`) that includes [ +permissions to manage access to the +project](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + +### gcloud CLI + +If you do not have gcloud CLI, install. For details, see the [Install the gcloud +CLI documentation](https://cloud.google.com/sdk/docs/install). + +### Google service account + +The tutorial assumes the use of a service account. If you do not have a service +account to use for this tutorial, create a service account. For details, see +[Create service +accounts](https://cloud.google.com/iam/docs/service-accounts-create#creating). + +### Terraform + +If you do not have Terraform installed, [install +Terraform](https://developer.hashicorp.com/terraform/install?product_intent=terraform). + +### kubectl and plugins + +{{< tip >}} + +Using `gcloud` to install `kubectl` will also install the needed plugins. +Otherwise, you will need to manually install the `gke-gcloud-auth-plugin` for +`kubectl`. + +{{< /tip >}} + +- If you do not have `kubectl`, install `kubectl`. To install, see [Install + kubectl and configure cluster + access](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) + for details. You will configure `kubectl` to interact with your GKE cluster + later in the tutorial. + +- If you do not have `gke-gcloud-auth-plugin` for `kubectl`, install the + `gke-gcloud-auth-plugin`. For details, see [Install the + gke-gcloud-auth-plugin](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_plugin). + +### Helm 3.2.0+ + +If you do not have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### jq (Optional) + +*Optional*. `jq` is used to parse the EKS cluster name and region from the +Terraform outputs. Alternatively, you can manually specify the name and region. +If you want to use `jq` and do not have `jq` installed, install. + +## Procedure + +### A. Setup GCP service account and authenticate + +1. Open a Terminal window. + +1. Initialize the gcloud CLI (`gcloud init`) to specify the GCP project you want + to use. For details, see the [Initializing the gcloud CLI + documentation](https://cloud.google.com/sdk/docs/initializing#initialize_the). + + {{< tip >}} + You do not need to configure a default Compute Region and Zone as you will + specify the region. + {{}} + +1. To the service account that will be used to perform the upgrade, + grant the following IAM roles (if the account does not have them already): + + - `roles/editor` + - `roles/iam.serviceAccountAdmin` + - `roles/storage.admin` + + 1. Enter your GCP project ID. + + ```bash + read -s PROJECT_ID + ``` + + 1. Find your service account email for your GCP project + + ```bash + gcloud iam service-accounts list --project $PROJECT_ID + ``` + + 1. Enter your service account email. + + ```bash + read -s SERVICE_ACCOUNT + ``` + + 1. Grant the service account the neccessary IAM roles. + + ```bash + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/editor" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/iam.serviceAccountAdmin" + + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:$SERVICE_ACCOUNT" \ + --role="roles/storage.admin" + ``` + +1. For the service account, authenticate to allow Terraform to interact with + your GCP project. For details, see [Terraform: Google Cloud Provider + Configuration + reference](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#authentication). + + For example, if using [User Application Default + Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default), + you can run the following command: + + ```bash + gcloud auth application-default login + ``` + + {{< tip >}} + If using `GOOGLE_APPLICATION_CREDENTIALS`, use absolute path to your key file. + {{}} + +### B. Upgrade Materialize operator and instances + +1. Go to the `examples/simple` folder in the Materialize Terraform repo + directory. + + ```bash + cd terraform-google-materialize/examples/simple + ``` + +1. Optional. You may need to update your fork of the Terraform module to + upgrade. + + {{< tip >}} + {{% self-managed/gcp-terraform-upgrade-notes %}} + + See [Materialize on GCP releases](/self-managed/installation/appendix-terraforms/#materialize-on-gcp-terraform-module) for notable changes. + + {{}} + +1. Configure `kubectl` to connect to your EKS cluster, specifying: + + - ``. Your cluster name has the form `-gke`; e.g., + `mz-simple-gke`. + + - ``. By default, the example Terraform module uses the `us-central1` + region. + + - ``. Your GCP project ID. + + ```bash + gcloud container clusters get-credentials \ + --region \ + --project + ``` + + Alternatively, you can use the following command to get the cluster name and + region from the Terraform output and the project ID from the environment + variable set earlier. + + ```bash + gcloud container clusters get-credentials $(terraform output -json gke_cluster | jq -r .name) \ + --region $(terraform output -json gke_cluster | jq -r .location) --project $PROJECT_ID + ``` + + To verify that you have configured correctly, run the following command: + + ```bash + kubectl cluster-info + ``` + + For help with `kubectl` commands, see [kubectl Quick + reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +{{% self-managed/versions/upgrade/upgrade-steps-cloud %}} diff --git a/doc/user/content/self-managed/installation/install-on-local-kind/_index.md b/doc/user/content/self-managed/installation/install-on-local-kind/_index.md new file mode 100644 index 0000000000000..c749b2b9a2325 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-local-kind/_index.md @@ -0,0 +1,310 @@ +--- +title: "Install locally on kind (via Helm)" +description: "Deploy Self-managed Materialize to a kind cluster." +aliases: + - /self-hosted/install-on-local-kind/ +menu: + main: + parent: "installation" + identifier: "install-on-local-kind" + weight: 10 +disable_list: true +--- + +{{% self-managed/materialize-components-sentence %}} + +The following tutorial uses a local [`kind`](https://kind.sigs.k8s.io/) cluster +and deploys the following components: + +- Materialize Operator using Helm into your local `kind` cluster. +- MinIO object storage as the blob storage for your Materialize. +- PostgreSQL database as the metadata database for your Materialize. +- Materialize as a containerized application into your local `kind` cluster. + +{{< important >}} + +This tutorial is for local evaluation/testing purposes only. + +- The tutorial uses sample configuration files that are for evaluation/testing + purposes only. +- The tutorial uses a Kubernetes metrics server with TLS disabled. In practice, + refer to your organization's official security practices. + +{{< /important >}} + +## Prerequisites + +### kind + +Install [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start/). + +### Docker + +Install [`Docker`](https://docs.docker.com/get-started/get-docker/). + +#### Docker resource requirements + +{{% self-managed/local-resource-requirements %}} + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### `kubectl` + +This tutorial uses `kubectl`. To install, refer to the [`kubectl` +documentationq](https://kubernetes.io/docs/tasks/tools/). + +For help with `kubectl` commands, see [kubectl Quick +reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +## Installation + +1. Start Docker if it is not already running. + + {{% self-managed/local-resource-requirements %}} + +1. Open a Terminal window. + +1. Create a working directory and go to the directory. + + ```shell + mkdir my-local-mz + cd my-local-mz + ``` + +1. Create a `kind` cluster. + + ```shell + kind create cluster + ``` + +1. Add labels `materialize.cloud/disk=true`, `materialize.cloud/swap=true` and + `workload=materialize-instance` to the `kind` node (in this example, the + `kind-control-plane` node). + + ```shell + MYNODE=$(kubectl get nodes --no-headers | awk '{print $1}') + kubectl label node $MYNODE materialize.cloud/disk=true + kubectl label node $MYNODE materialize.cloud/swap=true + kubectl label node $MYNODE workload=materialize-instance + ``` + + Verify that the labels were successfully applied by running the following + command: + + ```shell + kubectl get nodes --show-labels + ``` + +1. To help you get started for local evaluation/testing, Materialize provides + some sample configuration files. Download the sample configuration files from + the Materialize repo: + + {{% self-managed/versions/curl-sample-files-local-install %}} + +1. Install the Materialize Helm chart. + + 1. Add the Materialize Helm chart repository. + + ```shell + helm repo add materialize https://materializeinc.github.io/materialize + ``` + + 1. Update the repository. + + ```shell + helm repo update materialize + ``` + + {{% self-managed/versions/step-install-helm-version-local-kind-install %}} + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/my-materialize-operator-6c4c7d6fc9-hbzvr 1/1 Running 0 16s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/my-materialize-operator 1/1 1 1 16s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/my-materialize-operator-6c4c7d6fc9 1 1 1 16s + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting) guide. + +1. Install PostgreSQL and MinIO. + + 1. Use the `sample-postgres.yaml` file to install PostgreSQL as the + metadata database: + + ```shell + kubectl apply -f sample-postgres.yaml + ``` + + 1. Use the `sample-minio.yaml` file to install MinIO as the blob storage: + + ```shell + kubectl apply -f sample-minio.yaml + ``` + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/minio-777db75dd4-zcl89 1/1 Running 0 84s + pod/my-materialize-operator-6c4c7d6fc9-hbzvr 1/1 Running 0 107s + pod/postgres-55fbcd88bf-b4kdv 1/1 Running 0 86s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/minio ClusterIP 10.96.51.9 9000/TCP 84s + service/postgres ClusterIP 10.96.19.166 5432/TCP 86s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/minio 1/1 1 1 84s + deployment.apps/my-materialize-operator 1/1 1 1 107s + deployment.apps/postgres 1/1 1 1 86s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/minio-777db75dd4 1 1 1 84s + replicaset.apps/my-materialize-operator-6c4c7d6fc9 1 1 1 107s + replicaset.apps/postgres-55fbcd88bf 1 1 1 86s + ``` + +1. Install the metrics service to the `kube-system` namespace. + + 1. Add the metrics server Helm repository. + + ```shell + helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ + ``` + + 1. Update the repository. + + ```shell + helm repo update metrics-server + ``` + + 1. Install the metrics server to the `kube-system` namespace. + + {{< important >}} + + This tutorial is for local evaluation/testing purposes only. For simplicity, + the tutorial uses a Kubernetes metrics server with TLS disabled. In practice, + refer to your organization's official security practices. + + {{< /important >}} + + ```shell + helm install metrics-server metrics-server/metrics-server \ + --namespace kube-system \ + --set args="{--kubelet-insecure-tls,--kubelet-preferred-address-types=InternalIP\,Hostname\,ExternalIP}" + ``` + + You can verify the installation by running the following command: + + ```bash + kubectl get pods -n kube-system -l app.kubernetes.io/instance=metrics-server + ``` + + Wait for the `metrics-server` pod to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + metrics-server-89dfdc559-bq59m 1/1 Running 0 2m6s + ``` + +1. Install Materialize into a new `materialize-environment` namespace: + + 1. Use the `sample-materialize.yaml` file to create the + `materialize-environment` namespace and install Materialize: + + ```shell + kubectl apply -f sample-materialize.yaml + ``` + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize-environment + ``` + + Wait for the components to be ready and in the `Running` state. + + ```none + NAME READY STATUS RESTARTS AGE + pod/mz32bsnzerqo-balancerd-756b65959c-6q9db 1/1 Running 0 12s + pod/mz32bsnzerqo-cluster-s2-replica-s1-gen-1-0 1/1 Running 0 14s + pod/mz32bsnzerqo-cluster-u1-replica-u1-gen-1-0 1/1 Running 0 14s + pod/mz32bsnzerqo-console-6b7c975fb9-jkm8l 1/1 Running 0 5s + pod/mz32bsnzerqo-console-6b7c975fb9-z8g8f 1/1 Running 0 5s + pod/mz32bsnzerqo-environmentd-1-0 1/1 Running 0 19s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/mz32bsnzerqo-balancerd ClusterIP None 6876/TCP,6875/TCP 12s + service/mz32bsnzerqo-cluster-s2-replica-s1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 14s + service/mz32bsnzerqo-cluster-u1-replica-u1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 14s + service/mz32bsnzerqo-console ClusterIP None 8080/TCP 5s + service/mz32bsnzerqo-environmentd ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 12s + service/mz32bsnzerqo-environmentd-1 ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 19s + service/mz32bsnzerqo-persist-pubsub-1 ClusterIP None 6879/TCP 19s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/mz32bsnzerqo-balancerd 1/1 1 1 12s + deployment.apps/mz32bsnzerqo-console 2/2 2 2 5s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/mz32bsnzerqo-balancerd-756b65959c 1 1 1 12s + replicaset.apps/mz32bsnzerqo-console-6b7c975fb9 2 2 2 5s + + NAME READY AGE + statefulset.apps/mz32bsnzerqo-cluster-s2-replica-s1-gen-1 1/1 14s + statefulset.apps/mz32bsnzerqo-cluster-u1-replica-u1-gen-1 1/1 14s + statefulset.apps/mz32bsnzerqo-environmentd-1 1/1 19s + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-hosted/troubleshooting) guide. + +1. Open the Materialize Console in your browser: + + {{% self-managed/port-forwarding-handling-local %}} + + {{< tip >}} + + {{% self-managed/troubleshoot-console-mz_catalog_server_blurb %}} + + {{< /tip >}} + +## Next steps + +{{% self-managed/next-steps %}} + +## Clean up + +To delete the whole local deployment (including Materialize instances and data): + +```bash +kind delete cluster +``` + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-local-kind/upgrade-on-local-kind.md b/doc/user/content/self-managed/installation/install-on-local-kind/upgrade-on-local-kind.md new file mode 100644 index 0000000000000..59a18af0b87ba --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-local-kind/upgrade-on-local-kind.md @@ -0,0 +1,61 @@ +--- +title: "Upgrade on kind" +description: "Upgrade Materialize running locally on a kind cluster." +menu: + main: + parent: "install-on-local-kind" + identifier: "upgrade-on-local-kind" +--- + +To upgrade your Materialize instances, upgrade the Materialize operator first +and then the Materialize instances. The following tutorial upgrades your +Materialize deployment running locally on a [`kind`](https://kind.sigs.k8s.io/) +cluster. + +The tutorial assumes you have installed Materialize on `kind` using the +instructions on [Install locally on kind](/self-managed/installation/install-on-local-kind/). + +## Version compatibility + +When updating, you need to specify the Materialize Operator version, +`orchestratord` version, and the `environmentd` versions. The following table +presents the versions compatibility for the operator and the applications: + +{{< yaml-table data="self_managed/self_managed_operator_compatibility" >}} + +## Prerequisites + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### `kubectl` + +This tutorial uses `kubectl`. To install, refer to the [`kubectl` +documentationq](https://kubernetes.io/docs/tasks/tools/). + +For help with `kubectl` commands, see [kubectl Quick +reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +## Upgrade + +{{< important >}} + +The following procedure performs an in-place upgrade, which incurs downtime. + +To perform a rolling upgrade(where both the old and new Materialize instances +are running before the the old instances are removed), you can specify +`inPlaceRollout` to false. When performing a rolling upgrade, ensure you have +enough resources to support having both the old and new Materialize instances +running. + +{{}} + +{{% self-managed/versions/upgrade/upgrade-steps-local-kind %}} + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-local-minikube/_index.md b/doc/user/content/self-managed/installation/install-on-local-minikube/_index.md new file mode 100644 index 0000000000000..0f7dbb961803d --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-local-minikube/_index.md @@ -0,0 +1,313 @@ +--- +title: "Install locally on minikube (via Helm)" +description: "Deploy Self-managed Materialize to a local minikube cluster." +aliases: + - /self-hosted/install-on-local-minikube/ +menu: + main: + parent: "installation" + identifier: "install-on-local-minikube" + weight: 15 +disable_list: true +--- + +{{% self-managed/materialize-components-sentence %}} + +The following tutorial deploys the following components onto your local +[`minikube`](https://minikube.sigs.k8s.io/docs/start/): + +- Materialize Operator using Helm into your local `minikube` cluster. +- MinIO object storage as the blob storage for your Materialize. +- PostgreSQL database as the metadata database for your Materialize. +- Materialize as a containerized application into your local `minikube` cluster. + +{{< important >}} + +This tutorial is for local evaluation/testing purposes only. + +- The tutorial uses sample configuration files that are for evaluation/testing + purposes only. +- The tutorial uses a Kubernetes metrics server with TLS disabled. In practice, + refer to your organization's official security practices. + +{{< /important >}} + +## Prerequisites + +### minikube + +Install [`minikube`](https://minikube.sigs.k8s.io/docs/start/). + +### Container or virtual machine manager + +The following tutorial uses `Docker` as the container or virtual machine +manager. To use another container or virtual machine manager as listed on the +[`minikube` documentation](https://minikube.sigs.k8s.io/docs/start/), refer to +the specific container/VM manager documentation. + +Install [`Docker`](https://docs.docker.com/get-started/get-docker/). + +#### Resource requirements for your container/virtual machine manager + +{{% self-managed/local-resource-requirements %}} + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### `kubectl` + +This tutorial uses `kubectl`. To install, refer to the [`kubectl` documentationq](https://kubernetes.io/docs/tasks/tools/). + +For help with `kubectl` commands, see [kubectl Quick +reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +## Installation + +1. Start Docker if it is not already running. + + {{% self-managed/local-resource-requirements %}} + +1. Open a Terminal window. + +1. Create a working directory and go to the directory. + + ```shell + mkdir my-local-mz + cd my-local-mz + ``` + +1. Create a minikube cluster. + + ```shell + minikube start + ``` + +1. Add labels `materialize.cloud/disk=true`, `materialize.cloud/swap=true` and + `workload=materialize-instance` to the `minikube` node (in this example, + named `minikube`). + + ```shell + MYNODE=$(kubectl get nodes --no-headers | awk '{print $1}') + kubectl label node $MYNODE materialize.cloud/disk=true + kubectl label node $MYNODE materialize.cloud/swap=true + kubectl label node $MYNODE workload=materialize-instance + ``` + + Verify that the labels were successfully applied by running the following command: + + ```shell + kubectl get nodes --show-labels + ``` + +1. To help you get started for local evaluation/testing, Materialize provides + some sample configuration files. Download the sample configuration files from + the Materialize repo: + + {{% self-managed/versions/curl-sample-files-local-install %}} + +1. Install the Materialize Helm chart. + + 1. Add the Materialize Helm chart repository. + + ```shell + helm repo add materialize https://materializeinc.github.io/materialize + ``` + + 1. Update the repository. + + ```shell + helm repo update materialize + ``` + + {{% self-managed/versions/step-install-helm-version-local-minikube-install %}} + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/my-materialize-operator-7c75785df9-6cn88 1/1 Running 0 9s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/my-materialize-operator 1/1 1 1 9s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/my-materialize-operator-7c75785df9 1 1 1 9s + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-managed/installation/troubleshooting) guide. + +1. Install PostgreSQL and MinIO. + + 1. Use the `sample-postgres.yaml` file to install PostgreSQL as the + metadata database: + + ```shell + kubectl apply -f sample-postgres.yaml + ``` + + 1. Use the `sample-minio.yaml` file to install MinIO as the blob storage: + + ```shell + kubectl apply -f sample-minio.yaml + ``` + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize + ``` + + Wait for the components to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + pod/minio-777db75dd4-7cd77 1/1 Running 0 30s + pod/my-materialize-operator-7c75785df9-6cn88 1/1 Running 0 88s + pod/postgres-55fbcd88bf-zkwvz 1/1 Running 0 34s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/minio ClusterIP 10.98.114.67 9000/TCP 30s + service/postgres ClusterIP 10.98.144.251 5432/TCP 34s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/minio 1/1 1 1 30s + deployment.apps/my-materialize-operator 1/1 1 1 88s + deployment.apps/postgres 1/1 1 1 34s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/minio-777db75dd4 1 1 1 30s + replicaset.apps/my-materialize-operator-7c75785df9 1 1 1 88s + replicaset.apps/postgres-55fbcd88bf 1 1 1 34s + ``` + +1. Install the metrics service to the `kube-system` namespace. + + 1. Add the metrics server Helm repository. + + ```shell + helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ + ``` + + 1. Update the repository. + + ```shell + helm repo update metrics-server + ``` + + 1. Install the metrics server to the `kube-system` namespace. + + {{< important >}} + + This tutorial is for local evaluation/testing purposes only. For simplicity, + the tutorial uses a Kubernetes metrics server with TLS disabled. In practice, + refer to your organization's official security practices. + + {{< /important >}} + + ```shell + helm install metrics-server metrics-server/metrics-server \ + --namespace kube-system \ + --set args="{--kubelet-insecure-tls,--kubelet-preferred-address-types=InternalIP\,Hostname\,ExternalIP}" + ``` + + You can verify the installation by running the following command: + + ```bash + kubectl get pods -n kube-system -l app.kubernetes.io/instance=metrics-server + ``` + + Wait for the `metrics-server` pod to be ready and in the `Running` state: + + ```none + NAME READY STATUS RESTARTS AGE + metrics-server-89dfdc559-jt94n 1/1 Running 0 14m + ``` + +1. Install Materialize into a new `materialize-environment` namespace: + + 1. Use the `sample-materialize.yaml` file to create the + `materialize-environment` namespace and install Materialize: + + ```shell + kubectl apply -f sample-materialize.yaml + ``` + + 1. Verify the installation and check the status: + + ```shell + kubectl get all -n materialize-environment + ``` + + Wait for the components to be ready and in the `Running` state. + + ```none + NAME READY STATUS RESTARTS AGE + pod/mz10wiu7fyr7-balancerd-fddc4bd7c-lwqmp 1/1 Running 0 19s + pod/mz10wiu7fyr7-cluster-s2-replica-s1-gen-1-0 1/1 Running 0 26s + pod/mz10wiu7fyr7-cluster-u1-replica-u1-gen-1-0 1/1 Running 0 25s + pod/mz10wiu7fyr7-console-6cbcd997dc-95tf2 1/1 Running 0 12s + pod/mz10wiu7fyr7-console-6cbcd997dc-bbm5h 1/1 Running 0 12s + pod/mz10wiu7fyr7-environmentd-1-0 1/1 Running 0 32s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/mz10wiu7fyr7-balancerd ClusterIP None 6876/TCP,6875/TCP 19s + service/mz10wiu7fyr7-cluster-s2-replica-s1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 26s + service/mz10wiu7fyr7-cluster-u1-replica-u1-gen-1 ClusterIP None 2100/TCP,2103/TCP,2101/TCP,2102/TCP,6878/TCP 26s + service/mz10wiu7fyr7-console ClusterIP None 8080/TCP 12s + service/mz10wiu7fyr7-environmentd ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 19s + service/mz10wiu7fyr7-environmentd-1 ClusterIP None 6875/TCP,6876/TCP,6877/TCP,6878/TCP 32s + service/mz10wiu7fyr7-persist-pubsub-1 ClusterIP None 6879/TCP 32s + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/mz10wiu7fyr7-balancerd 1/1 1 1 19s + deployment.apps/mz10wiu7fyr7-console 2/2 2 2 12s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/mz10wiu7fyr7-balancerd-fddc4bd7c 1 1 1 19s + replicaset.apps/mz10wiu7fyr7-console-6cbcd997dc 2 2 2 12s + + NAME READY AGE + statefulset.apps/mz10wiu7fyr7-cluster-s2-replica-s1-gen-1 1/1 26s + statefulset.apps/mz10wiu7fyr7-cluster-u1-replica-u1-gen-1 1/1 26s + statefulset.apps/mz10wiu7fyr7-environmentd-1 1/1 32s + ``` + + If you run into an error during deployment, refer to the + [Troubleshooting](/self-hosted/troubleshooting) guide. + +1. Open the Materialize Console in your browser: + + {{% self-managed/port-forwarding-handling-local %}} + + {{< tip >}} + + {{% self-managed/troubleshoot-console-mz_catalog_server_blurb %}} + + {{< /tip >}} + +## Next steps + +{{% self-managed/next-steps %}} + +## Clean up + +To delete the whole local deployment (including Materialize instances and data): + +```bash +minikube delete +``` + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/install-on-local-minikube/upgrade-on-local-minikube.md b/doc/user/content/self-managed/installation/install-on-local-minikube/upgrade-on-local-minikube.md new file mode 100644 index 0000000000000..1a6de236232f6 --- /dev/null +++ b/doc/user/content/self-managed/installation/install-on-local-minikube/upgrade-on-local-minikube.md @@ -0,0 +1,60 @@ +--- +title: "Upgrade on minikube" +description: "Upgrade Materialize running locally on a minikube cluster." +menu: + main: + parent: "install-on-local-minikube" +--- + +To upgrade your Materialize instances, upgrade the Materialize operator first +and then the Materialize instances. The following tutorial upgrades your +Materialize deployment running locally on a +[`minikube`](https://minikube.sigs.k8s.io/docs/start/) cluster. + +The tutorial assumes you have installed Materialize on `minikube` using the +instructions on [Install locally on minikube](/self-managed/installation/install-on-local-minikube/). + +## Version compatibility + +When updating, you need to specify the Materialize Operator version, +`orchestratord` version, and the `environmentd` versions. The following table +presents the versions compatibility for the operator and the applications: + +{{< yaml-table data="self_managed/self_managed_operator_compatibility" >}} + +## Prerequisites + +### Helm 3.2.0+ + +If you don't have Helm version 3.2.0+ installed, install. For details, see the +[Helm documentation](https://helm.sh/docs/intro/install/). + +### `kubectl` + +This tutorial uses `kubectl`. To install, refer to the [`kubectl` documentationq](https://kubernetes.io/docs/tasks/tools/). + +For help with `kubectl` commands, see [kubectl Quick +reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +## Upgrade + + +{{< important >}} + +The following procedure performs an in-place upgrade, which incurs downtime. + +To perform a rolling upgrade(where both the old and new Materialize instances +are running before the the old instances are removed), you can specify +`inPlaceRollout` to false. When performing a rolling upgrade, ensure you have +enough resources to support having both the old and new Materialize instances +running. + +{{}} + +{{% self-managed/versions/upgrade/upgrade-steps-local-minikube %}} + +## See also + +- [Materialize Operator Configuration](/self-managed/installation/configuration/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/operational-guidelines.md b/doc/user/content/self-managed/installation/operational-guidelines.md new file mode 100644 index 0000000000000..71710424bde88 --- /dev/null +++ b/doc/user/content/self-managed/installation/operational-guidelines.md @@ -0,0 +1,64 @@ +--- +title: "Operational guidelines" +description: "" +aliases: + - /self-hosted/operational-guidelines/ +menu: + main: + parent: "installation" + weight: 80 + identifier: "sm-operational-guidelines" +--- + +## Recommended instance types + +- ARM-based CPU +- 1:8 ratio of vCPU to GiB memory (if spill-to-disk is not enabled) +- 1:16 ratio of vCPU to GiB local instance storage (if spill-to-disk is enabled) + +See also the specific cloud provider guidance: + +- [AWS Deployment + guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/#recommended-instance-types) + +- [GCP Deployment + guidelines](/self-managed/installation/install-on-gcp/appendix-deployment-guidelines/#recommended-instance-types) + +- [Azure Deployment + guidelines](/self-managed/installation/install-on-azure/appendix-deployment-guidelines/#recommended-instance-types) + +## CPU affinity + +It is strongly recommended to enable the Kubernetes `static` [CPU management policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy). +This ensures that each worker thread of Materialize is given exclusively access to a vCPU. Our benchmarks have shown this +to substantially improve the performance of compute-bound workloads. + +## TLS + +When running with TLS in production, run with certificates from an official +Certificate Authority (CA) rather than self-signed certificates. + +## Locally-attached NVMe storage + +For optimal performance, Materialize requires fast, locally-attached NVMe +storage. Having a locally-attached storage allows Materialize to spill to disk +when operating on datasets larger than main memory as well as allows for a more +graceful degradation rather than OOMing. Network-attached storage (like EBS +volumes) can significantly degrade performance and is not supported. + +Refer to the specific cloud provider guidelines: + +- [AWS Deployment + guidelines](/self-managed/installation/install-on-aws/appendix-deployment-guidelines/) + +- [GCP Deployment + guidelines](/self-managed/installation/install-on-gcp/appendix-deployment-guidelines/) + +- [Azure Deployment + guidelines](/self-managed/installation/install-on-azure/appendix-deployment-guidelines/) + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/installation/troubleshooting.md b/doc/user/content/self-managed/installation/troubleshooting.md new file mode 100644 index 0000000000000..f1dc0aeaf54b9 --- /dev/null +++ b/doc/user/content/self-managed/installation/troubleshooting.md @@ -0,0 +1,133 @@ +--- +title: "Troubleshooting" +description: "" +aliases: + - /self-hosted/troubleshooting/ +menu: + main: + parent: "installation" + weight: 90 +--- + +## Troubleshooting Kubernetes + +### Materialize operator + +To check the status of the Materialize operator: + +```shell +kubectl -n materialize get all +``` + +If you encounter issues with the Materialize operator, + +- Check the operator logs, using the label selector: + + ```shell + kubectl -n materialize logs -l app.kubernetes.io/name=materialize-operator + ``` + +- Check the log of a specific object (pod/deployment/etc) running in + your namespace: + + ```shell + kubectl -n materialize logs / + ``` + + In case of a container restart, to get the logs for previous instance, include + the `--previous` flag. + +- Check the events for the operator pod: + + - You can use `kubectl describe`, substituting your pod name for ``: + + ```shell + kubectl -n materialize describe pod/ + ``` + + - You can use `kubectl get events`, substituting your pod name for + ``: + + ```shell + kubectl -n materialize get events --sort-by=.metadata.creationTimestamp --field-selector involvedObject.name= + ``` + +### Materialize deployment + +- To check the status of your Materialize deployment, run: + + ```shell + kubectl -n materialize-environment get all + ``` + +- To check the log of a specific object (pod/deployment/etc) running in your + namespace: + + ```shell + kubectl -n materialize-environment logs / + ``` + + In case of a container restart, to get the logs for previous instance, include + the `--previous` flag. + +- To describe an object, you can use `kubectl describe`: + + ```shell + kubectl -n materialize-environment describe / + ``` + +For additional `kubectl` commands, see [kubectl Quick reference](https://kubernetes.io/docs/reference/kubectl/quick-reference/). + +## Troubleshooting Console unresponsiveness + +If you experience long loading screens or unresponsiveness in the Materialize +Console, it may be that the size of the `mz_catalog_server` cluster (where the +majority of the Console's queries are run) is insufficient (default size is +`25cc`). + +To increase the cluster's size, you can follow the following steps: + +1. Login as the `mz_system` user in order to update `mz_catalog_server`. + + 1. To login as `mz_system` you'll need the internal-sql port found in the + `environmentd` pod (`6877` by default). You can port forward via `kubectl + port-forward svc/mzXXXXXXXXXX 6877:6877 -n materialize-environment`. + + 1. Connect using a pgwire compatible client (e.g., `psql`) and connect using + the port and user `mz_system`. For example: + + ``` + psql -h localhost -p 6877 --user mz_system + ``` + +3. Run the following [ALTER CLUSTER](/sql/alter-cluster/#resizing) statement + to change the cluster size to `50cc`: + + ```mzsql + ALTER CLUSTER mz_catalog_server SET (SIZE = '50cc'); + ``` + +4. Verify your changes via `SHOW CLUSTERS;` + + ```mzsql + show clusters; + ``` + + The output should include the `mz_catalog_server` cluster with a size of `50cc`: + + ```none + name | replicas | comment + -------------------+-----------+--------- + mz_analytics | | + mz_catalog_server | r1 (50cc) | + mz_probe | | + mz_support | | + mz_system | | + quickstart | r1 (25cc) | + (6 rows) + ``` + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) diff --git a/doc/user/content/self-managed/installation/upgrading.md b/doc/user/content/self-managed/installation/upgrading.md new file mode 100644 index 0000000000000..41b74b68eacc5 --- /dev/null +++ b/doc/user/content/self-managed/installation/upgrading.md @@ -0,0 +1,169 @@ +--- +title: "Upgrading" +description: "Upgrading Helm chart and Materialize." +menu: + main: + parent: "installation" +draft: true +--- + +The following provides steps for upgrading the Materialize operator and +Materialize instances. While the operator and instances can be upgraded +independently, you should ensure version compatibility between them. + +When upgrading: + +- Ensure version compatibility between the Materialize operator and Materialize + instance. The operator can manage instances within a certain version range + version range. + +- Upgrade the operator first. + +- Always upgrade your Materialize instances after upgrading the operator to + ensure compatibility. + +### Upgrading the Helm Chart + +{{< important >}} + +Upgrade the operator first. + +{{}} + +To upgrade the Materialize operator to a new version: + +```shell +helm upgrade my-materialize-operator materialize/misc/helm-charts/operator +``` + +If you have custom values, make sure to include your values file: + +```shell +helm upgrade my-materialize-operator materialize/misc/helm-charts/operator -f my-values.yaml +``` + +### Upgrading Materialize Instances + +{{< important >}} + +Always upgrade your Materialize instances after upgrading the operator to +ensure compatibility. + +{{}} + +To upgrade your Materialize instances, you'll need to update the Materialize custom resource and trigger a rollout. + +By default, the operator performs rolling upgrades (`inPlaceRollout: false`) which minimize downtime but require additional Kubernetes cluster resources during the transition. However, keep in mind that rolling upgrades typically take longer to complete due to the sequential rollout process. For environments where downtime is acceptable, you can opt for in-place upgrades (`inPlaceRollout: true`). + +#### Determining the Version + +The compatible version for your Materialize instances is specified in the Helm chart's `appVersion`. For the installed chart version, you can run: + +```shell +helm list -n materialize +``` + +Or check the `Chart.yaml` file in the `misc/helm-charts/operator` directory: + +```yaml +apiVersion: v2 +name: materialize-operator +# ... +version: 25.1.0-beta.1 +appVersion: v0.125.2 # Use this version for your Materialize instances +``` + +Use the `appVersion` (`v0.125.2` in this case) when updating your Materialize instances to ensure compatibility. + +#### Using `kubectl` patch + +For standard upgrades such as image updates: + +```shell +# For version updates, first update the image reference +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"environmentdImageRef\": \"materialize/environmentd:v0.125.2\"}}" + +# Then trigger the rollout with a new UUID +kubectl patch materialize \ + -n \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\"}}" +``` + +You can combine both operations in a single command if preferred: + +```shell +kubectl patch materialize 12345678-1234-1234-1234-123456789012 \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"environmentdImageRef\": \"materialize/environmentd:v0.125.2\", \"requestRollout\": \"$(uuidgen)\"}}" +``` + +#### Using YAML Definition + +Alternatively, you can update your Materialize custom resource definition directly: + +```yaml +apiVersion: materialize.cloud/v1alpha1 +kind: Materialize +metadata: + name: 12345678-1234-1234-1234-123456789012 + namespace: materialize-environment +spec: + environmentdImageRef: materialize/environmentd:v0.125.2 # Update version as needed + requestRollout: 22222222-2222-2222-2222-222222222222 # Generate new UUID + forceRollout: 33333333-3333-3333-3333-333333333333 # Optional: for forced rollouts + inPlaceRollout: false # When false, performs a rolling upgrade rather than in-place + backendSecretName: materialize-backend +``` + +Apply the updated definition: + +```shell +kubectl apply -f materialize.yaml +``` + +#### Forced Rollouts + +If you need to force a rollout even when there are no changes to the instance: + +```shell +kubectl patch materialize \ + -n materialize-environment \ + --type='merge' \ + -p "{\"spec\": {\"requestRollout\": \"$(uuidgen)\", \"forceRollout\": \"$(uuidgen)\"}}" +``` + +The behavior of a forced rollout follows your `inPlaceRollout` setting: +- With `inPlaceRollout: false` (default): Creates new instances before terminating the old ones, temporarily requiring twice the resources during the transition +- With `inPlaceRollout: true`: Directly replaces the instances, causing downtime but without requiring additional resources + +### Verifying the Upgrade + +After initiating the rollout, you can monitor the status: + +```shell +# Watch the status of your Materialize environment +kubectl get materialize -n materialize-environment -w + +# Check the logs of the operator +kubectl logs -l app.kubernetes.io/name=materialize-operator -n materialize +``` + +### Notes on Rollouts + +- `requestRollout` triggers a rollout only if there are actual changes to the instance (like image updates) +- `forceRollout` triggers a rollout regardless of whether there are changes, which can be useful for debugging or when you need to force a rollout for other reasons +- Both fields expect UUID values and each rollout requires a new, unique UUID value +- `inPlaceRollout`: + - When `false` (default): Performs a rolling upgrade by spawning new instances before terminating old ones. While this minimizes downtime, there may still be a brief interruption during the transition. + - When `true`: Directly replaces existing instances, which will cause downtime. + +## See also + +- [Configuration](/self-managed/installation/configuration/) +- [Installation](/self-managed/installation/) +- [Troubleshooting](/self-managed/installation/troubleshooting/) diff --git a/doc/user/content/self-managed/v25.2/_index.md b/doc/user/content/self-managed/v25.2/_index.md deleted file mode 100644 index ae83e78e841de..0000000000000 --- a/doc/user/content/self-managed/v25.2/_index.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Self-managed Materialize" -description: "" -aliases: - - /self-managed/ - - /lts/sql/create-source/file/ - - /lts/sql/create-source/kafka/ - - /lts/sql/types/float/ - - /lts/sql/create-source/s3/ - - /lts/sql/types/integer/ - - /lts/sql/show-index/ - - /lts/overview/timelines/ - - /lts/sql/create-source/kinesis/ - - /lts/integrations/sql-clients/ - - /lts/sql/create-source/pubnub/ - - /lts/sql/types/timestamp/ - - /lts/ - - /lts/sql/create-source/materialize-cdc/ - - /lts/install/ - - /lts/ops/monitoring/ - - /lts/overview/architecture/ - - /lts/integrations/aws-aurora/ - - /lts/overview/arrangements/ - - /lts/sql/create-source/ - - /lts/sql/functions/date-bin-hopping/ - - /lts/sql/functions/list_agg/ - - /lts/sql/patterns/top-k/ - - /lts/sql/types/array/ - - /lts/sql/types/text/ - - /lts/get-started/ - - /lts/integrations/debezium/ - - /lts/license/ - - /lts/overview/ - - /lts/sql/ - - /lts/sql/alter-index/ - - /lts/sql/execute/ - - /lts/sql/functions/csv_extract/ - - /lts/sql/functions/justify-days/ - - /lts/sql/prepare/ - - /lts/sql/show-columns/ - - /lts/sql/show-create-sink/ - - /lts/sql/system-catalog/ - - /lts/sql/types/interval/ - - /lts/support/ - - /lts/integrations/ - - /lts/integrations/aws-msk/ - - /lts/integrations/aws-rds/ - - /lts/integrations/dbt/ - - /lts/integrations/golang/ - - /lts/integrations/metabase/ - - /lts/ops/optimization/ - - /lts/ops/troubleshooting/ - - /lts/overview/key-concepts/ - - /lts/quickstarts/live-analytics/ - - /lts/quickstarts/log-parsing/ - - /lts/sql/commit/ - - /lts/sql/copy-from/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/_index.md b/doc/user/content/self-managed/v25.2/installation/_index.md deleted file mode 100644 index 8390c15868674..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Installation" -description: "Installation guides for self-managed Materialize." -disable_list: true -aliases: - - /self-managed/installation/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/configuration.md b/doc/user/content/self-managed/v25.2/installation/configuration.md deleted file mode 100644 index ff086bf308ad1..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/configuration.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Materialize Operator Configuration" -description: "" -aliases: - - /self-hosted/configuration/ - - /self-managed/configuration/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-aws.md b/doc/user/content/self-managed/v25.2/installation/install-on-aws.md deleted file mode 100644 index 8c94084270466..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-aws.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install on AWS" -description: "" -aliases: - - /self-managed/installation/install-on-aws/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md b/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md deleted file mode 100644 index 7badbb5930455..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install on GCP" -description: "" -aliases: - - /self-managed/installation/install-on-gcp/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md b/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md deleted file mode 100644 index 25f8b8b7b5186..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Install locally on kind" -description: "" -aliases: - - /self-hosted/install-on-local-kind/ - - /self-managed/installation/install-on-local-kind/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md b/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md deleted file mode 100644 index 1f275c2158dbf..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install locally on minikube" -description: "" -aliases: - - /self-managed/installation/install-on-local-minikube/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md b/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md deleted file mode 100644 index 160da0f2cac39..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Operational guidelines" -description: "" -robots: "noindex, nofollow" -aliases: - - /self-managed/operational-guidelines/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/troubleshooting.md b/doc/user/content/self-managed/v25.2/installation/troubleshooting.md deleted file mode 100644 index 1b8d2346afd9e..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/troubleshooting.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Troubleshooting" -description: "" -aliases: - - /self-hosted/troubleshooting/ - - /self-managed/troubleshooting/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/upgrading.md b/doc/user/content/self-managed/v25.2/installation/upgrading.md deleted file mode 100644 index b2d8cfeb6082e..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/upgrading.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Upgrading" -description: "Upgrading Helm chart and Materialize." -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/serve-results/_index.md b/doc/user/content/serve-results/_index.md index efcc704d0085e..6751b1d5f5b93 100644 --- a/doc/user/content/serve-results/_index.md +++ b/doc/user/content/serve-results/_index.md @@ -4,7 +4,7 @@ description: "Serving results from Materialize" disable_list: true menus: main: - weight: 13 + weight: 15 identifier: 'serve-results' --- diff --git a/doc/user/content/serve-results/bi-tools/looker.md b/doc/user/content/serve-results/bi-tools/looker.md index 3942db8138c9e..09faee3055f06 100644 --- a/doc/user/content/serve-results/bi-tools/looker.md +++ b/doc/user/content/serve-results/bi-tools/looker.md @@ -60,7 +60,7 @@ When using Looker with Materialize, be aware of the following limitations: ``` b. Via the Materialize Console: - - Go to [Materialize Console](https://console.materialize.com/) + - Go to [Materialize Console](/console/) - Navigate to Query History - Filter by 'Running' queries - Click on the query you want to cancel diff --git a/doc/user/content/serve-results/sink/census.md b/doc/user/content/serve-results/sink/census.md index 8000719e1c9fd..064bafc4ba35b 100644 --- a/doc/user/content/serve-results/sink/census.md +++ b/doc/user/content/serve-results/sink/census.md @@ -28,7 +28,7 @@ To begin you will need to add your Materialize database as a source in Census. 1. From the list of connection types, choose **Materialize**. -1. Set the connection parameters using the credentials provided in the [Materialize console](https://console.materialize.com/). +1. Set the connection parameters using the credentials provided in the [Materialize console](/console/). Then click the **Connect** button. ## Step 2. Set up a Destination diff --git a/doc/user/content/serve-results/sink/s3.md b/doc/user/content/serve-results/sink/s3.md index 00e088e27520d..04535da2470f5 100644 --- a/doc/user/content/serve-results/sink/s3.md +++ b/doc/user/content/serve-results/sink/s3.md @@ -146,7 +146,7 @@ Next, you must attach the policy you just created to a Materialize-specific ## Step 2. Create a connection -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, create an [AWS connection](/sql/create-connection/#aws), replacing `` with the 12-digit number that identifies your AWS account, and `` with the name of the role you created in the diff --git a/doc/user/content/serve-results/sink/s3_compatible.md b/doc/user/content/serve-results/sink/s3_compatible.md index 9e31834f41ece..d8efbe6e88fd2 100644 --- a/doc/user/content/serve-results/sink/s3_compatible.md +++ b/doc/user/content/serve-results/sink/s3_compatible.md @@ -22,7 +22,7 @@ Cloud Storage, or Cloudflare R2. ## Step 1. Create a connection -1. In the [SQL Shell](https://console.materialize.com/), or your preferred SQL +1. In the [SQL Shell](/console/), or your preferred SQL client connected to Materialize, create an [AWS connection](/sql/create-connection/#aws), replacing `` and `` with the credentials for your bucket. The AWS connection can be used to connect to any S3 compatible object storage service, by specifying the endpoint and the region. diff --git a/doc/user/content/sql/alter-sink.md b/doc/user/content/sql/alter-sink.md index 1ffd33cfcf71d..c86df643cdf68 100644 --- a/doc/user/content/sql/alter-sink.md +++ b/doc/user/content/sql/alter-sink.md @@ -29,7 +29,7 @@ subsequent execution of the sink will result in errors and will not be able to make progress. To monitor the status of a sink after an `ALTER SINK` command, navigate to the -respective object page in the [Materialize console](https://console.materialize.com/), +respective object page in the [Materialize console](/console/), or query the [`mz_internal.mz_sink_statuses`](/sql/system-catalog/mz_internal/#mz_sink_statuses) system catalog view. diff --git a/doc/user/content/sql/create-connection.md b/doc/user/content/sql/create-connection.md index 60c8e153b9461..43b22132b4d45 100644 --- a/doc/user/content/sql/create-connection.md +++ b/doc/user/content/sql/create-connection.md @@ -217,7 +217,7 @@ connections to create [sources](/sql/create-source/kafka) and [sinks](/sql/creat | `AWS CONNECTION` | object name | The name of an [AWS connection](#aws) to use when performing IAM authentication with an Amazon MSK cluster.

Only valid if the security protocol is `SASL_PLAINTEXT` or `SASL_SSL`. | `AWS PRIVATELINK` | object name | The name of an [AWS PrivateLink connection](#aws-privatelink) to route network traffic through.

Exactly one of `BROKER`, `BROKERS`, or `AWS PRIVATELINK` must be specified. | `PROGRESS TOPIC` | `text` | The name of a topic that Kafka sinks can use to track internal consistency metadata. Default: `_materialize-progress-{REGION ID}-{CONNECTION ID}`. -| `PROGRESS TOPIC REPLICATION FACTOR` | `int` | {{< warn-if-unreleased-inline "v0.106" >}} The partition count to use when creating the progress topic (if the Kafka topic does not already exist).
Default: Broker's default. +| `PROGRESS TOPIC REPLICATION FACTOR` | `int` | The partition count to use when creating the progress topic (if the Kafka topic does not already exist).
Default: Broker's default. #### `WITH` options {#kafka-with-options} diff --git a/doc/user/content/sql/create-sink/kafka.md b/doc/user/content/sql/create-sink/kafka.md index 1167763eaf4db..8e173c566c895 100644 --- a/doc/user/content/sql/create-sink/kafka.md +++ b/doc/user/content/sql/create-sink/kafka.md @@ -60,7 +60,7 @@ _item_name_ | The name of the source, table or materialized view you want **KEY (** _key_column_ **)** | An optional list of columns to use as the Kafka message key. If unspecified, the Kafka key is left unset. **HEADERS** | An optional column containing headers to add to each Kafka message emitted by the sink. See [Headers](#headers) for details. **FORMAT** | Specifies the format to use for both keys and values: `AVRO USING csr_connection`, `JSON`, `TEXT`, or `BYTES`. See [Formats](#formats) for details. -**KEY FORMAT .. VALUE FORMAT** | {{< warn-if-unreleased-inline "v0.108" >}} Specifies the key format and value formats separately. See [Formats](#formats) for details. +**KEY FORMAT .. VALUE FORMAT** | Specifies the key format and value formats separately. See [Formats](#formats) for details. **NOT ENFORCED** | Whether to disable validation of key uniqueness when using the upsert envelope. See [Upsert key selection](#upsert-key-selection) for details. **ENVELOPE DEBEZIUM** | The generated schemas have a [Debezium-style diff envelope](#debezium-envelope) to capture changes in the input view or source. **ENVELOPE UPSERT** | The sink emits data with [upsert semantics](#upsert-envelope). diff --git a/doc/user/content/sql/create-source/load-generator.md b/doc/user/content/sql/create-source/load-generator.md index 49658635128b3..4a27b3d4bcb16 100644 --- a/doc/user/content/sql/create-source/load-generator.md +++ b/doc/user/content/sql/create-source/load-generator.md @@ -36,8 +36,8 @@ _src_name_ | The name for the source. **TPCH** | Use the [tpch](#tpch) load generator. **IF NOT EXISTS** | Do nothing (except issuing a notice) if a source with the same name already exists. **TICK INTERVAL** | The interval at which the next datum should be emitted. Defaults to one second. -**AS OF** | The tick at which to start producing data. Defaults to 0. {{< warn-if-unreleased-inline "v0.101" >}} -**UP TO** | The tick before which to stop producing data. Defaults to infinite. {{< warn-if-unreleased-inline "v0.101" >}} +**AS OF** | The tick at which to start producing data. Defaults to 0. +**UP TO** | The tick before which to stop producing data. Defaults to infinite. **SCALE FACTOR** | The scale factor for the `TPCH` generator. Defaults to `0.01` (~ 10MB). **FOR ALL TABLES** | Creates subsources for all tables in the load generator. **EXPOSE PROGRESS AS** _progress_subsource_name_ | The name of the progress subsource for the source. If this is not specified, the subsource will be named `_progress`. For more information, see [Monitoring source progress](#monitoring-source-progress). diff --git a/doc/user/content/sql/create-source/webhook.md b/doc/user/content/sql/create-source/webhook.md index 572758dec5061..5b1a4ec9dcd2f 100644 --- a/doc/user/content/sql/create-source/webhook.md +++ b/doc/user/content/sql/create-source/webhook.md @@ -82,7 +82,7 @@ https:///api/webhook/// A breakdown of each component is as follows: -- ``: The Materialize instance URL, which can be found on the [Materialize console](https://console.materialize.com/). +- ``: The Materialize instance URL, which can be found on the [Materialize console](/console/). - ``: The name of the database where the source is created (default is `materialize`). - ``: The schema name where the source gets created (default is `public`). - ``: The name you provided for your source at the time of creation. diff --git a/doc/user/content/sql/select/recursive-ctes.md b/doc/user/content/sql/select/recursive-ctes.md index a3c070915f423..c8251666597cc 100644 --- a/doc/user/content/sql/select/recursive-ctes.md +++ b/doc/user/content/sql/select/recursive-ctes.md @@ -74,7 +74,7 @@ However, introducing recursive CTEs complicates the situation as follows: Let's consider a very simple schema consisting of `users` that belong to a hierarchy of geographical `areas` and exchange `transfers` between each other. -Use the [SQL Shell](https://console.materialize.com/) to run the sequence of +Use the [SQL Shell](/console/) to run the sequence of commands below. diff --git a/doc/user/content/sql/system-catalog/mz_catalog.md b/doc/user/content/sql/system-catalog/mz_catalog.md index caaeda13b60c9..1ec37b4b83d5c 100644 --- a/doc/user/content/sql/system-catalog/mz_catalog.md +++ b/doc/user/content/sql/system-catalog/mz_catalog.md @@ -65,8 +65,6 @@ Field | Type | Meaning ### `mz_cluster_replica_frontiers` -{{< warn-if-unreleased "v0.118" >}} - The `mz_cluster_replica_frontiers` table describes the per-replica frontiers of sources, sinks, materialized views, indexes, and subscriptions in the system, as observed from the coordinator. @@ -296,7 +294,6 @@ Field | Type | Meaning ### `mz_kafka_sources` -{{< warn-if-unreleased v0.115 >}} The `mz_kafka_sources` table contains a row for each Kafka source in the system. This table was previously in the `mz_internal` schema. All queries previously referencing @@ -417,8 +414,6 @@ Field | Type | Meaning ### `mz_recent_storage_usage` -{{< warn-if-unreleased "v0.113" >}} - The `mz_recent_storage_usage` table describes the storage utilization of each table, source, and materialized view in the system in the most recent storage utilization assessment. Storage utilization assessments occur approximately diff --git a/doc/user/content/sql/system-catalog/mz_internal.md b/doc/user/content/sql/system-catalog/mz_internal.md index 6923de28fdbee..26237a7d9bc2f 100644 --- a/doc/user/content/sql/system-catalog/mz_internal.md +++ b/doc/user/content/sql/system-catalog/mz_internal.md @@ -174,7 +174,6 @@ At this time, we do not make any guarantees about the exactness or freshness of ## `mz_cluster_replica_metrics_history` -{{< warn-if-unreleased v0.116 >}} The `mz_cluster_replica_metrics_history` table records resource utilization metrics for all processes of all extant cluster replicas. @@ -208,7 +207,6 @@ of each process in each cluster replica in the system. ## `mz_cluster_replica_status_history` -{{< warn-if-unreleased v0.116 >}} The `mz_cluster_replica_status_history` table records status changes for all processes of all extant cluster replicas. @@ -240,7 +238,6 @@ At this time, we do not make any guarantees about the exactness or freshness of ## `mz_cluster_replica_utilization_history` -{{< warn-if-unreleased v0.116 >}} The `mz_cluster_replica_utilization_history` view records resource utilization metrics for all processes of all extant cluster replicas, as a percentage of the total resource allocation. diff --git a/doc/user/content/sql/types/list.md b/doc/user/content/sql/types/list.md index 9f655c92a9ee0..7249a297fa94c 100644 --- a/doc/user/content/sql/types/list.md +++ b/doc/user/content/sql/types/list.md @@ -521,8 +521,6 @@ Like [array containment operators in PostgreSQL](https://www.postgresql.org/docs list containment operators in Materialize **do not** account for duplicates. {{< /note >}} -{{< warn-if-unreleased "v0.107" >}} - ```mzsql SELECT LIST[1,4,3] @> LIST[3,1] AS contains; ``` diff --git a/doc/user/content/support.md b/doc/user/content/support.md index 1bf6ab585d340..ac0b80eca0238 100644 --- a/doc/user/content/support.md +++ b/doc/user/content/support.md @@ -35,8 +35,9 @@ you can use the **Report Production Outage** option for fastest response. ### Email -You can also email us your questions at `support@materialize.com` from your -Materialize login email address. +You can also email us your questions at `support@materialize.com`. For +Materialize Cloud SLA Supported Customer, email from your Materialize login +email address. ### Join the Materialize Slack community @@ -49,9 +50,11 @@ If you are an SLA Supported Customer who is experiencing an **urgent** issue, - Use the [messenger's](#messenger-app) **Report Production Outage** option for fastest response. -- Alternatively, you can email `support@materialize.com` from your Materialize - login email address. To ensure priority handling, include **urgent** in the - subject line. +- Alternatively, you can email `support@materialize.com`. To ensure priority + handling, include **urgent** in the subject line. + + - For Materialize Cloud SLA Supported Customer, email from your Materialize + login email address. Use the following definitions to determine what priority to assign to your support ticket: @@ -104,6 +107,7 @@ Have ideas to improve Materialize? Use the messenger to start a conversation — We tailor support experiences based on your workspace and plan. To learn more about what's available to your team, reach out via the messenger or email us. -## Checking service status +## Checking Materialize Cloud service status -Check the current status of our services on the [status page](https://status.materialize.com). +Check the current status of Materialize Cloud on the [status +page](https://status.materialize.com). diff --git a/doc/user/content/transform-data/dataflow-troubleshooting.md b/doc/user/content/transform-data/dataflow-troubleshooting.md index 5c5f238bdb9d2..2fe34e3e3a627 100644 --- a/doc/user/content/transform-data/dataflow-troubleshooting.md +++ b/doc/user/content/transform-data/dataflow-troubleshooting.md @@ -381,7 +381,7 @@ ORDER BY mas.size DESC; 16318559 | ArrangeBy[[Column(0)]]-errors | Dataflow: materialize.public.num_bids_idx | 0 | 0 bytes ``` -In the [Materialize Console](https://console.materialize.com), +In the [Materialize Console](/console/), - The [**Cluster Overview**](/console/clusters/) page displays the cluster resource utilization for a selected cluster as well as the resource intensive diff --git a/doc/user/content/transform-data/troubleshooting.md b/doc/user/content/transform-data/troubleshooting.md index c36fec03b9d26..d920e8668f249 100644 --- a/doc/user/content/transform-data/troubleshooting.md +++ b/doc/user/content/transform-data/troubleshooting.md @@ -37,7 +37,7 @@ This is how Materialize ensures consistent results for all queries. To check if any materialized views or indexes are lagging, use the workflow graphs in the Materialize console. -1. Go to https://console.materialize.com/. +1. Go to /console/. 2. Click on the **"Clusters"** tab in the side navigation bar. 3. Click on the cluster that contains your upstream materialized view or index. 4. Go to the **"Materialized Views"** or **"Indexes"** section, and click on the @@ -183,7 +183,7 @@ on, taking longer to return. As an example, if you issue a lot of resource-intensive queries at once, that might spike the CPU. The measure of cluster busyness is CPU. You can monitor CPU usage in the -[Materialize console](https://console.materialize.com/) by clicking +[Materialize console](/console/) by clicking the **"Clusters"** tab in the navigation bar, and clicking into the cluster. You can also grab CPU usage from the system catalog using SQL: @@ -232,7 +232,7 @@ Hydration time is proportional to data volume and query complexity. This means that you should expect objects with large volumes of data and/or complex queries to take longer to hydrate. You should also expect hydration to be triggered every time a cluster is restarted or sized up, including during -[Materialize's routine maintenance window](/releases#schedule). +[Materialize's routine maintenance window](/releases/#release-schedules). ### Unhealthy cluster @@ -244,7 +244,7 @@ If your cluster replica is CPU-maxed out (~100% CPU usage), your query may be bl We recommend setting [Alerting thresholds](https://materialize.com/docs/manage/monitor/alerting/#thresholds) to notify your team when a cluster is reaching its capacity. Please note that these are recommendations, and some configurations may reach unstable memory utilization levels sooner than the thresholds. -To see Memory Utilization and CPU usage for your cluster replica in the [Materialize console](https://materialize.com/docs/console/clusters/), go to [https://console.materialize.com/](https://console.materialize.com/), click the **“Clusters”** tab in the navigation bar, and click on the cluster name. +To see Memory Utilization and CPU usage for your cluster replica in the [Materialize console](https://materialize.com/docs/console/clusters/), go to [/console/](/console/), click the **“Clusters”** tab in the navigation bar, and click on the cluster name. #### Address @@ -274,7 +274,7 @@ attribute performance information to each LIR operator. Materialize stores a (sampled) log of the SQL statements that are issued against your Materialize region in the last **three days**, along with various metadata about these statements. You can access this log via the **"Query history"** tab -in the [Materialize console](https://console.materialize.com/). You can filter +in the [Materialize console](/console/). You can filter and sort statements by type, duration, and other dimensions. This data is also available via the diff --git a/doc/user/data/examples/ingest_data/mysql/create_source_cloud.yml b/doc/user/data/examples/ingest_data/mysql/create_source_cloud.yml new file mode 100644 index 0000000000000..58db542ce62dd --- /dev/null +++ b/doc/user/data/examples/ingest_data/mysql/create_source_cloud.yml @@ -0,0 +1,23 @@ +- name: "create-source" + description: | + Once you have created the connection, you can use the connection in the + [`CREATE SOURCE`](/sql/create-source/) command to connect to your MySQL instance and start ingesting + data: + code: | + CREATE SOURCE mz_source + FROM MYSQL CONNECTION mysql_connection + FOR ALL TABLES; + +- name: "create-source-options" + description: | + + - By default, the source will be created in the active cluster; to use a different cluster, use the `IN CLUSTER` clause. + + - To ingest data from specific schemas or tables, use the `FOR SCHEMAS (,)` or `FOR TABLES (, )` options instead of `FOR ALL TABLES`. + + - To handle [unsupported data types](#supported-types), use the `TEXT COLUMNS` or `IGNORE COLUMNS` options. + +- name: "schema-changes" + description: | + After source creation, refer to [schema changes + considerations](#schema-changes) for information on handling upstream schema changes. diff --git a/doc/user/data/examples/ingest_data/postgres/create_connection_ips_cloud.yml b/doc/user/data/examples/ingest_data/postgres/create_connection_ips_cloud.yml new file mode 100644 index 0000000000000..18759b1d7a354 --- /dev/null +++ b/doc/user/data/examples/ingest_data/postgres/create_connection_ips_cloud.yml @@ -0,0 +1,57 @@ +- name: "create-secret" + description: |- + In the [Materialize Console's SQL Shell](/console/), or your preferred SQL + client connected to Materialize, use the [`CREATE + SECRET`](/sql/create-secret/) command to securely store the password for the + `materialize` PostgreSQL user you created + [earlier](#2-create-a-publication-and-a-replication-user): + + code: | + CREATE SECRET pgpass AS ''; + +- name: "create-connection" + description: | + Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create a + connection object with access and authentication details for Materialize to + use: + + code: | + CREATE CONNECTION pg_connection TO POSTGRES ( + HOST '', + PORT 5432, + USER 'materialize', + PASSWORD SECRET pgpass, + SSL MODE 'require', + DATABASE '' + ); + +- name: "create-connection-options-general" + description: | + + - Replace `` with your PostgreSQL endpoint. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-aurora" + description: | + - Replace `` with the **Writer** endpoint for your Aurora database. To + find the endpoint, select your database in the AWS Management Console, + then click the **Connectivity & security** tab and look for the endpoint + with type **Writer**. + +
+ WARNING! + You must use the Writer endpoint for the database. Using a Reader endpoint will not work. +
+ + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-rds" + description: | + - Replace `` with your RDS endpoint. To find your RDS endpoint, select + your database in the RDS Console, and look under **Connect & security** + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. diff --git a/doc/user/data/examples/ingest_data/postgres/create_connection_privatelink_cloud.yml b/doc/user/data/examples/ingest_data/postgres/create_connection_privatelink_cloud.yml new file mode 100644 index 0000000000000..714ce2aac300f --- /dev/null +++ b/doc/user/data/examples/ingest_data/postgres/create_connection_privatelink_cloud.yml @@ -0,0 +1,169 @@ +- name: "create-privatelink-connection-aurora" + description: |- + In the [Materialize Console's SQL Shell](/console/), or your preferred SQL + client connected to Materialize, use the [`CREATE + CONNECTION`](/sql/create-connection/#aws-privatelink) command to create an + AWS PrivateLink connection: + code: | + CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( + SERVICE NAME 'com.amazonaws.vpce.us-east-1.vpce-svc-0356210a8a432d9e9', + AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4') + ); +- name: "create-privatelink-connection-options-aurora" + description: | + - Replace the `SERVICE NAME` value with the service name you noted + [earlier](#b-optional-configure-network-security). + + - Replace the `AVAILABILITY ZONES` list with the IDs of the availability + zones in your AWS account. + + To find your availability zone IDs, select your database in the RDS + Console and click the subnets under **Connectivity & security**. For each + subnet, look for **Availability Zone ID** (e.g., `use1-az6`), + not **Availability Zone** (e.g., `us-east-1d`). + +- name: "create-privatelink-connection-rds-intro" + description: |- + In the [SQL Shell](https://console.materialize.com/), or your preferred SQL + client connected to Materialize, use the [`CREATE + CONNECTION`](/sql/create-connection/#aws-privatelink) command to create an + **in-region** or **cross-region** AWS PrivateLink connection. +- name: "create-privatelink-connection-rds-in-region" + description: |- + ↕️ **In-region connections** + + To connect to an AWS PrivateLink endpoint service in the **same region** as + your Materialize environment: + + code: | + CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( SERVICE + NAME 'com.amazonaws.vpce..vpce-svc-', + AVAILABILITY ZONES ('use1-az1', 'use1-az2', 'use1-az4') ); + +- name: "create-privatelink-connection-rds-in-region-options" + description: | + - Replace the `SERVICE NAME` value with the service name you noted + [earlier](#b-optional-configure-network-security). + + - Replace the `AVAILABILITY ZONES` list with the IDs of the availability + zones in your AWS account. For in-region connections the availability zones + of the NLB and the consumer VPC **must match**. + + To find your availability zone IDs, select your database in the RDS + Console and click the subnets under **Connectivity & security**. For each + subnet, look for **Availability Zone ID** (e.g., `use1-az6`), not + **Availability Zone** (e.g., `us-east-1d`). + +- name: "create-privatelink-connection-rds-cross-region" + description: |- + ↔️ **Cross-region connections** + + To connect to an AWS PrivateLink endpoint service in a **different region** + to the one where your Materialize environment is deployed: + code: | + CREATE CONNECTION privatelink_svc TO AWS PRIVATELINK ( SERVICE + NAME 'com.amazonaws.vpce.us-west-1.vpce-svc-', -- For + now, the AVAILABILITY ZONES clause **is** required, but will be -- made + optional in a future release. AVAILABILITY ZONES () ); + +- name: "create-privatelink-connection-rds-cross-region-options" + description: |- + - Replace the `SERVICE NAME` value with the service name you noted + [earlier](#b-optional-configure-network-security). + + - The service name region refers to where the endpoint service was created. + You **do not need** to specify `AVAILABILITY ZONES` manually — these will be + optimally auto-assigned when none are provided. + +- name: "get-principal-privatelink-connection" + description: | + Retrieve the AWS principal for the AWS PrivateLink connection you just created: + + code: | + SELECT principal + FROM mz_aws_privatelink_connections plc + JOIN mz_connections c ON plc.id = c.id + WHERE c.name = 'privatelink_svc'; + + +- name: "get-principal-privatelink-connection-results" + description: | + The results should resemble: + ``` + principal + --------------------------------------------------------------------------- + arn:aws:iam::664411391173:role/mz_20273b7c-2bbe-42b8-8c36-8cc179e9bbc3_u1 + ``` + +- name: "update-vpc-endpoint" + description: | + Update your VPC endpoint service to [accept connections from the AWS principal](https://docs.aws.amazon.com/vpc/latest/privatelink/add-endpoint-service-permissions.html). + +- name: "approve-connection-request" + description: | + If your AWS PrivateLink service is configured to require acceptance of + connection requests, [manually approve the connection request from + Materialize](https://docs.aws.amazon.com/vpc/latest/privatelink/configure-endpoint-service.html#accept-reject-connection-requests). + + results: | + **Note:** It can take some time for the connection request to show up. Do + not move on to the next step until you've approved the connection. + +- name: "validate-connection" + description: | + Validate the AWS PrivateLink connection you created using the [`VALIDATE + CONNECTION`](/sql/validate-connection) command: + code: | + VALIDATE CONNECTION privatelink_svc; + results: | + If no validation error is returned, move to the next step. + +- name: "create-secret" + description: | + Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the + password for the `materialize` PostgreSQL user you created + [earlier](#2-create-a-publication-and-a-replication-user): + + code: | + CREATE SECRET pgpass AS ''; + +- name: "create-connection" + description: | + Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create + another connection object, this time with database access and authentication + details for Materialize to use: + + code: | + CREATE CONNECTION pg_connection TO POSTGRES ( + HOST '', + PORT 5432, + USER 'materialize', + PASSWORD SECRET pgpass, + DATABASE '', + AWS PRIVATELINK privatelink_svc + ); + +- name: "create-connection-options-aurora" + description: | + - Replace `` with your Aurora endpoint. To find your Aurora endpoint, + select your database in the AWS Management Console, and look + under **Connectivity & security**. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-rds" + description: | + - Replace `` with your RDS endpoint. To find your RDS endpoint, select + your database in the RDS Console, and look under **Connectivity & + security**. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-self-hosted" + description: | + - Replace `` with your database endpoint. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. diff --git a/doc/user/data/examples/ingest_data/postgres/create_connection_ssh_cloud.yml b/doc/user/data/examples/ingest_data/postgres/create_connection_ssh_cloud.yml new file mode 100644 index 0000000000000..4e7106433434e --- /dev/null +++ b/doc/user/data/examples/ingest_data/postgres/create_connection_ssh_cloud.yml @@ -0,0 +1,111 @@ +- name: "create-ssh-tunnel-connection" + description: |- + In the [Materialize Console's SQL Shell](/console/), or your preferred SQL + client connected to Materialize, use the [`CREATE + CONNECTION`](/sql/create-connection/#ssh-tunnel) command to create an SSH + tunnel connection: + + code: | + CREATE CONNECTION ssh_connection TO SSH TUNNEL ( + HOST '', + PORT , + USER '' + ); + +- name: "create-ssh-tunnel-connection-options" + description: |- + - Replace `` and `` with the public IP + address and port of the SSH bastion host you created + [earlier](#b-optional-configure-network-security). + + - Replace `` with the username for the key pair you + created for your SSH bastion host. + +- name: "get-public-keys-general" + description: | + Get Materialize's public keys for the SSH tunnel connection: + code: | + SELECT * FROM mz_ssh_tunnel_connections; + +- name: "get-public-keys-aurora-rds-self-hosted" + description: | + Get Materialize's public keys for the SSH tunnel connection: + code: | + SELECT + mz_connections.name, + mz_ssh_tunnel_connections.* + FROM + mz_connections + JOIN + mz_ssh_tunnel_connections USING(id) + WHERE + mz_connections.name = 'ssh_connection'; + +- name: "login-to-ssh-bastion-host" + description: | + Log in to your SSH bastion host and add Materialize's public keys to the + `authorized_keys` file, for example: + + code: | + echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys + echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys + +- name: "validate-ssh-tunnel-connection" + description: | + Back in the SQL client connected to Materialize, validate the SSH tunnel + connection you created using the [`VALIDATE + CONNECTION`](/sql/validate-connection) command: + + code: | + VALIDATE CONNECTION ssh_connection; + results: | + If no validation error is returned, move to the next step. + +- name: "create-secret" + description: | + Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the + password for the `materialize` PostgreSQL user you created + [earlier](#2-create-a-publication-and-a-replication-user): + + code: | + CREATE SECRET pgpass AS ''; + +- name: "create-connection" + description: | + + Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create another connection object, this time with database access and authentication details for Materialize to use: + + code: | + CREATE CONNECTION pg_connection TO POSTGRES ( + HOST '', + PORT 5432, + USER 'materialize', + PASSWORD SECRET pgpass, + DATABASE '', + SSH TUNNEL ssh_connection + ); + +- name: "create-connection-options-general" + description: | + - Replace `` with your PostgreSQL endpoint. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-aurora" + description: | + - Replace `` with your Aurora endpoint. To find your Aurora endpoint, + select your database in the AWS Management Console, and look under + **Connectivity & security**. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. + +- name: "create-connection-options-rds" + description: | + - Replace `` with your RDS endpoint. To find your RDS endpoint, + select your database in the RDS Console, and look under + **Connectivity & security**. + + - Replace `` with the name of the database containing the tables + you want to replicate to Materialize. diff --git a/doc/user/data/examples/ingest_data/postgres/create_source_cloud.yml b/doc/user/data/examples/ingest_data/postgres/create_source_cloud.yml new file mode 100644 index 0000000000000..4e8df7c5b2329 --- /dev/null +++ b/doc/user/data/examples/ingest_data/postgres/create_source_cloud.yml @@ -0,0 +1,26 @@ +- name: "create-source" + description: | + Once you have created the connection, you can use the connection in the + [`CREATE SOURCE`](/sql/create-source/) command to connect to your PostgreSQL + instance and start ingesting data from the publication you created + [earlier](#2-create-a-publication-and-a-replication-user): + + code: | + CREATE SOURCE mz_source + IN CLUSTER ingest_postgres + FROM POSTGRES CONNECTION pg_connection (PUBLICATION 'mz_source') + FOR ALL TABLES; + +- name: "create-source-options" + description: | + + - By default, the source will be created in the active cluster; to use a different cluster, use the `IN CLUSTER` clause. + + - To ingest data from specific schemas or tables, use the `FOR SCHEMAS (,)` or `FOR TABLES (, )` options instead of `FOR ALL TABLES`. + + - To handle [unsupported data types](#supported-types), use the `TEXT COLUMNS` or `IGNORE COLUMNS` options. + +- name: "schema-changes" + description: | + After source creation, refer to [schema changes + considerations](#schema-changes) for information on handling upstream schema changes. diff --git a/doc/user/data/rbac/recommendations.yml b/doc/user/data/rbac/recommendations.yml index 2067e4a175000..3bceaca0b0644 100644 --- a/doc/user/data/rbac/recommendations.yml +++ b/doc/user/data/rbac/recommendations.yml @@ -4,9 +4,11 @@ least privilege. Grant only the minimum access necessary for users and service accounts to perform their duties. -- recommendation: "Restrict the assignment of **Organization Admin** role" +- recommendation: "Restrict the assignment of **Organization Admin** role +(Materialize Cloud)" description: | + {{< include-md file="shared-content/rbac/rbac-intro-disambiguation.md" >}} {{< include-md file="shared-content/rbac/org-admin-recommendation.md" >}} - recommendation: "Restrict the granting of `CREATEROLE` privilege" diff --git a/doc/user/data/self_managed/authentication_setting.yml b/doc/user/data/self_managed/authentication_setting.yml new file mode 100644 index 0000000000000..3f9c6b5b42e25 --- /dev/null +++ b/doc/user/data/self_managed/authentication_setting.yml @@ -0,0 +1,19 @@ +columns: + - column: "authenticatorKind Value" + - column: "Description" + +rows: +- "authenticatorKind Value": "**None**" + "Description": | + Disables authentication. All users are trusted based on their claimed + identity **without** any verification. **Default** +- "authenticatorKind Value": "**Password**" + "Description": | + Enables [password authentication](#configuring-password-authentication) for + users. When enabled, users must authenticate with their password. + + {{< tip >}} + When enabled, you must also set the `mz_system` user password in + `external_login_password_mz_system`. See [Configuring password + authentication](#configuring-password-authentication) for details. + {{}} diff --git a/doc/user/data/self_managed/aws_required_variables.yml b/doc/user/data/self_managed/aws_required_variables.yml new file mode 100644 index 0000000000000..6a8dcb01a3c8f --- /dev/null +++ b/doc/user/data/self_managed/aws_required_variables.yml @@ -0,0 +1,21 @@ +columns: + - column: "Variable" + +rows: +- "Variable": | + + ```hcl + namespace = + ``` + +- "Variable": | + + ```hcl + environment = + ``` + +- "Variable": | + + ```hcl + database_password = + ``` diff --git a/doc/user/data/self_managed/aws_terraform_deployed_components.yml b/doc/user/data/self_managed/aws_terraform_deployed_components.yml new file mode 100644 index 0000000000000..88440a5ceec35 --- /dev/null +++ b/doc/user/data/self_managed/aws_terraform_deployed_components.yml @@ -0,0 +1,33 @@ +columns: + - column: "Component" + - column: "Version" + +rows: + - Component: "Kubernetes (EKS) cluster" + Version: "All" + - Component: "Dedicated VPC" + Version: "All" + - Component: "S3 for blob storage" + Version: "All" + - Component: "RDS PostgreSQL cluster and database" + Version: "All" + - Component: "Materialize Operator" + Version: "All" + - Component: | + Materialize instances (Deployed during subsequent runs after the Operator is running) + Version: "All" + - Component: | + AWS Load Balancer Controller and Network Load Balancers for each + Materialize instance + Version: | + [v0.3.0+](/self-managed/installation/appendix-terraforms/#materialize-on-aws-terraform-module) + - Component: | + OpenEBS and NVMe instance storage to enable spill-to-disk + Version: | + [v0.3.1+](/self-managed/installation/appendix-terraforms/#materialize-on-aws-terraform-module) + - Component: | + `cert-manager` and a self-signed `ClusterIssuer`. + `ClusterIssuer` is + deployed on subsequent runs after the `cert-manager` is running. + Version: | + [v0.4.0+](/self-managed/installation/appendix-terraforms/#materialize-on-aws-terraform-module) diff --git a/doc/user/data/self_managed/aws_terraform_versions.yml b/doc/user/data/self_managed/aws_terraform_versions.yml new file mode 100644 index 0000000000000..9fa583b20c882 --- /dev/null +++ b/doc/user/data/self_managed/aws_terraform_versions.yml @@ -0,0 +1,45 @@ +columns: + - column: "Terraform version" + - column: "Notable changes" + +rows: +- "Terraform version": | + [v0.5.5](https://github.com/MaterializeInc/terraform-aws-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.26. + +- "Terraform version": | + [v0.5.4](https://github.com/MaterializeInc/terraform-aws-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.25. + +- "Terraform version": | + [v0.4.9](https://github.com/MaterializeInc/terraform-aws-materialize/releases/) + "Notable changes": | + + - Uses `terraform-helm-materialize` v0.1.19. + - Bumps Materialize release to [self-managed 25.2](/releases/#self-managed-releases) + - Adds support for password authentication and enabling RBAC + +- "Terraform version": | + [v0.4.6](https://github.com/MaterializeInc/terraform-aws-materialize/releases/) + "Notable changes": | + + - Adds support for passing in additional Materialize instance configuration + options via [`environmentd_extra_args`](https://github.com/MaterializeInc/terraform-aws-materialize?tab=readme-ov-file#input_materialize_instances) + + {{% include-md file="shared-content/materialized_extra_args.md" %}} + + - Uses `terraform-helm-materialize` v0.1.15. + +- "Terraform version": | + [v0.4.5](https://github.com/MaterializeInc/terraform-aws-materialize/releases/tag/v0.4.5) + "Notable changes": | + - Defaults to using Materialize Operator v25.1.12 (via + `terraform-helm-materialize` v0.1.14). + +- "Terraform version": | + [v0.4.4](https://github.com/MaterializeInc/terraform-aws-materialize/releases/tag/v0.4.4) + "Notable changes": | + - Defaults to using Materialize Operator v25.1.11 (via + `terraform-helm-materialize` v0.1.13). diff --git a/doc/user/data/self_managed/azure_required_variables.yml b/doc/user/data/self_managed/azure_required_variables.yml new file mode 100644 index 0000000000000..eb304a932062d --- /dev/null +++ b/doc/user/data/self_managed/azure_required_variables.yml @@ -0,0 +1,34 @@ +columns: + - column: "Variable" + +rows: + +- "Variable": | + + ```hcl + resource_group_name = + ``` + +- "Variable": | + + ```hcl + database_config = { + password = # required + # sku_name = # optional + # postgres_version = # optional + # username = # optional + # db_name = # optional + } + ``` + +- "Variable": | + + ```hcl + network_config = { # required starting in v0.2.0 + vnet_address_space = string + subnet_cidr = string + postgres_subnet_cidr = string + service_cidr = string + docker_bridge_cidr = string + } + ``` diff --git a/doc/user/data/self_managed/azure_terraform_deployed_components.yml b/doc/user/data/self_managed/azure_terraform_deployed_components.yml new file mode 100644 index 0000000000000..8c1648e21750b --- /dev/null +++ b/doc/user/data/self_managed/azure_terraform_deployed_components.yml @@ -0,0 +1,34 @@ +columns: + - column: "Component" + - column: "Version" + +rows: + - Component: "AKS cluster" + Version: "All" + - Component: "Azure Blob Storage for persistence" + Version: "All" + - Component: "Azure Database for PostgreSQL Flexible Server for metadata storage" + Version: "All" + - Component: "Required networking and security configurations" + Version: "All" + - Component: "Managed identities with proper RBAC permissions" + Version: "All" + - Component: "Materialize Operator" + Version: "All" + - Component: | + Materialize instances (Deployed during subsequent runs after the Operator is running) + Version: "All" + - Component: | + `cert-manager` and a self-signed `ClusterIssuer`. + `ClusterIssuer` is + deployed on subsequent runs after the `cert-manager` is running. + Version: | + [v0.3.0+](/self-managed/installation/appendix-terraforms/#materialize-on-azure-terraform-module) + - Component: | + Load balancers for each Materialize instance + Version: | + [v0.3.1+](/self-managed/installation/appendix-terraforms/#materialize-on-azure-terraform-module) + - Component: | + OpenEBS and NVMe instance storage to enable spill-to-disk + Version: | + [v0.4.0+](/self-managed/installation/appendix-terraforms/#materialize-on-azure-terraform-module) diff --git a/doc/user/data/self_managed/azure_terraform_versions.yml b/doc/user/data/self_managed/azure_terraform_versions.yml new file mode 100644 index 0000000000000..828e7350b6a31 --- /dev/null +++ b/doc/user/data/self_managed/azure_terraform_versions.yml @@ -0,0 +1,40 @@ +columns: + - column: "Terraform version" + - column: "Notable changes" + +rows: +- "Terraform version": | + [v0.5.5](https://github.com/MaterializeInc/terraform-azurerm-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.26. + +- "Terraform version": | + [v0.5.4](https://github.com/MaterializeInc/terraform-azurerm-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.25. + +- "Terraform version": | + [v0.4.6](https://github.com/MaterializeInc/terraform-azurerm-materialize/releases/) + "Notable changes": | + + - Uses `terraform-helm-materialize` v0.1.19. + - Bumps Materialize release to [self-managed 25.2](/releases/#self-managed-releases) + - Adds support for password authentication and enabling RBAC + +- "Terraform version": | + [v0.4.3](https://github.com/MaterializeInc/terraform-azurerm-materialize/releases/) + "Notable changes": | + + - Adds support for passing in additional Materialize instance configuration + options via + [`environmentd_extra_args`](https://github.com/MaterializeInc/terraform-azurerm-materialize?tab=readme-ov-file#input_materialize_instances) + + {{% include-md file="shared-content/materialized_extra_args.md" %}} + + - Uses `terraform-helm-materialize` v0.1.15. + +- "Terraform version": | + [v0.4.1](https://github.com/MaterializeInc/terraform-azurerm-materialize/releases/tag/v0.4.1) + "Notable changes": | + - Defaults to using Materialize Operator v25.1.11 (via + `terraform-helm-materialize` v0.1.13). diff --git a/doc/user/data/self_managed/gcp_required_variables.yml b/doc/user/data/self_managed/gcp_required_variables.yml new file mode 100644 index 0000000000000..5d62fe30cf7f4 --- /dev/null +++ b/doc/user/data/self_managed/gcp_required_variables.yml @@ -0,0 +1,37 @@ +columns: + - column: "Variable" + +rows: +- "Variable": | + + ```hcl + project_id = + ``` + +- "Variable": | + + ```hcl + prefix = + ``` + +- "Variable": | + + ```hcl + database_config = { + password = # required + # tier = # optional + # version = # optional + # username = # optional + # db_name = # optional + } + ``` + +- "Variable": | + + ```hcl + network_config = { # required starting in v0.3.0 + subnet_cidr = + pods_cidr = + services_cidr = + } + ``` diff --git a/doc/user/data/self_managed/gcp_terraform_deployed_components.yml b/doc/user/data/self_managed/gcp_terraform_deployed_components.yml new file mode 100644 index 0000000000000..fb7472ad308af --- /dev/null +++ b/doc/user/data/self_managed/gcp_terraform_deployed_components.yml @@ -0,0 +1,37 @@ +columns: + - column: "Component" + - column: "Version" + +rows: + - Component: "Google Kubernetes Engine (GKE) cluster" + Version: "All" + - Component: "Cloud Storage bucket for blob storage" + + Version: "All" + - Component: "Cloud SQL PostgreSQL database for metadata storage" + Version: "All" + - Component: "Dedicated VPC" + Version: "All" + - Component: "Service accounts with proper IAM permissions" + Version: "All" + - Component: "Materialize Operator" + Version: "All" + - Component: | + Materialize instances (Deployed during subsequent runs after the Operator is running) + Version: "All" + - Component: | + Load balancers for each Materialize instance + Version: | + [v0.3.0+](/self-managed/installation/appendix-terraforms/#materialize-on-gcp-terraform-module) + + - Component: | + `cert-manager` and a self-signed `ClusterIssuer`. + `ClusterIssuer` is + deployed on subsequent runs after the `cert-manager` is running. + Version: | + [v0.3.0+](/self-managed/installation/appendix-terraforms/#materialize-on-gcp-terraform-module) + + - Component: | + OpenEBS and NVMe instance storage to enable spill-to-disk + Version: | + [v0.4.0+](/self-managed/installation/appendix-terraforms/#materialize-on-gcp-terraform-module) diff --git a/doc/user/data/self_managed/gcp_terraform_versions.yml b/doc/user/data/self_managed/gcp_terraform_versions.yml new file mode 100644 index 0000000000000..44876a93b9b36 --- /dev/null +++ b/doc/user/data/self_managed/gcp_terraform_versions.yml @@ -0,0 +1,41 @@ +columns: + - column: "Terraform version" + - column: "Notable changes" + +rows: +- "Terraform version": | + [v0.5.5](https://github.com/MaterializeInc/terraform-google-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.26. + +- "Terraform version": | + [v0.5.4](https://github.com/MaterializeInc/terraform-google-materialize/releases/) + "Notable changes": | + - Uses `terraform-helm-materialize` v0.1.25. + +- "Terraform version": | + [v0.4.6](https://github.com/MaterializeInc/terraform-google-materialize/releases/) + "Notable changes": | + + - Uses `terraform-helm-materialize` v0.1.19. + - Bumps Materialize release to [self-managed 25.2](/releases/#self-managed-releases) + - Adds support for password authentication and enabling RBAC + +- "Terraform version": | + [v0.4.3](https://github.com/MaterializeInc/terraform-google-materialize/releases/) + "Notable changes": | + + - Adds support for passing in additional Materialize instance configuration + options via + [`environmentd_extra_args`](https://github.com/MaterializeInc/terraform-google-materialize?tab=readme-ov-file#input_materialize_instances) + + {{% include-md file="shared-content/materialized_extra_args.md" %}} + + - Uses `terraform-helm-materialize` v0.1.15. + +- "Terraform version": | + [v0.4.1](https://github.com/MaterializeInc/terraform-google-materialize/releases/tag/v0.4.1) + + "Notable changes": | + - Defaults to using Materialize Operator v25.1.11 (via + `terraform-helm-materialize` v0.1.13). diff --git a/doc/user/data/self_managed/latest_versions.yml b/doc/user/data/self_managed/latest_versions.yml new file mode 100644 index 0000000000000..9348d529ea62e --- /dev/null +++ b/doc/user/data/self_managed/latest_versions.yml @@ -0,0 +1,7 @@ +terraform_gcp_version: v0.5.5 +terraform_azure_version: v0.5.5 +terraform_aws_version: v0.5.5 +terraform_helm_version: v0.1.26 +operator_helm_chart_version: v25.2.11 +orchestratord_version: v0.159.1 +environmentd_version: v0.147.17 diff --git a/doc/user/data/self_managed/materialize_operator_chart_parameter.yml b/doc/user/data/self_managed/materialize_operator_chart_parameter.yml index e2fbcf4630c18..023b092b304c5 100644 --- a/doc/user/data/self_managed/materialize_operator_chart_parameter.yml +++ b/doc/user/data/self_managed/materialize_operator_chart_parameter.yml @@ -1,199 +1,231 @@ parameters: + - parameter: balancerd.enabled + description: | + Flag to indicate whether to create balancerd pods for the environments + default: "true" + + - parameter: balancerd.nodeSelector + description: | + Node selector to use for balancerd pods spawned by the operator + default: "{}" + - parameter: clusterd.nodeSelector description: | Node selector to use for clusterd pods spawned by the operator default: "{}" + + - parameter: console.enabled + description: | + Flag to indicate whether to create console pods for the environments + default: "true" + + - parameter: console.imageTagMapOverride + description: | + Override the mapping of environmentd versions to console versions + default: "{}" + + - parameter: console.nodeSelector + description: | + Node selector to use for console pods spawned by the operator + default: "{}" + - parameter: environmentd.nodeSelector description: | Node selector to use for environmentd pods spawned by the operator default: "{}" - parameter: networkPolicies.egress - description: egress from Materialize pods to sources and sinks - default: | - {"cidrs":["0.0.0.0/0"],"enabled":false} + description: | + egress from Materialize pods to sources and sinks + default: '{"cidrs":["0.0.0.0/0"],"enabled":false}' - parameter: networkPolicies.enabled description: | Whether to enable network policies for securing communication between pods - default: false + default: "false" - parameter: networkPolicies.ingress description: | - Ingress to the SQL and HTTP interfaces on environmentd or balancerd - default: | - {"cidrs":["0.0.0.0/0"],"enabled":false} + Whether to enable ingress to the SQL and HTTP interfaces on environmentd or balancerd + default: '{"cidrs":["0.0.0.0/0"],"enabled":false}' - parameter: networkPolicies.internal description: | - Internal communication between Materialize pods - default: | - {"enabled":false} + Whether to enable internal communication between Materialize pods + default: '{"enabled":false}' - parameter: observability.enabled - description: "" - default: true + description: | + Whether to enable observability features + default: "true" - parameter: observability.podMetrics.enabled description: | Whether to enable the pod metrics scraper which populates the Environment Overview Monitoring tab in the web console (requires metrics-server to be installed) - default: false + default: "false" - parameter: observability.prometheus.scrapeAnnotations.enabled description: | Whether to annotate pods with common keys used for prometheus scraping. - default: true + default: "true" - parameter: operator.args.enableInternalStatementLogging description: "" - default: true + default: "true" - parameter: operator.args.startupLogFilter - description: Log filtering settings for startup logs - default: INFO,mz_orchestratord=TRACE + description: | + Log filtering settings for startup logs + default: "INFO,mz_orchestratord=TRACE" - parameter: operator.cloudProvider.providers.aws.accountID - description: When using AWS, accountID is required + description: | + When using AWS, accountID is required default: "" - parameter: operator.cloudProvider.providers.aws.enabled description: "" - default: false + default: "false" - parameter: operator.cloudProvider.providers.aws.iam.roles.connection - description: ARN for CREATE CONNECTION feature + description: | + ARN for CREATE CONNECTION feature default: "" - parameter: operator.cloudProvider.providers.aws.iam.roles.environment - description: ARN of the IAM role for environmentd + description: | + ARN of the IAM role for environmentd default: "" - parameter: operator.cloudProvider.providers.gcp - description: GCP Configuration (placeholder for future use) - default: | - {"enabled":false} + description: | + GCP Configuration (placeholder for future use) + default: '{"enabled":false}' - parameter: operator.cloudProvider.region - description: Common cloud provider settings - default: | - kind + description: | + Common cloud provider settings + default: "kind" - parameter: operator.cloudProvider.type + description: | + Specifies cloud provider. Valid values are 'aws', 'gcp', 'azure', 'generic', or 'local' + default: "local" + + - parameter: operator.clusters.defaultReplicationFactor.analytics + description: "" + default: "0" + + - parameter: operator.clusters.defaultReplicationFactor.probe + description: "" + default: "0" + + - parameter: operator.clusters.defaultReplicationFactor.support + description: "" + default: "0" + + - parameter: operator.clusters.defaultReplicationFactor.system description: "" - default: local + default: "0" - parameter: operator.clusters.defaultSizes.analytics description: "" - default: 25cc + default: "25cc" - parameter: operator.clusters.defaultSizes.catalogServer description: "" - default: 50cc + default: "25cc" - parameter: operator.clusters.defaultSizes.default description: "" - default: 25cc + default: "25cc" - parameter: operator.clusters.defaultSizes.probe description: "" - default: mz_probe + default: "mz_probe" - parameter: operator.clusters.defaultSizes.support description: "" - default: 25cc + default: "25cc" - parameter: operator.clusters.defaultSizes.system description: "" - default: 25cc - - - parameter: operator.features.authentication - description: | - Whether to enable environmentd rbac checks. - - *Not yet supported in the helm chart* - default: false - - - parameter: operator.features.consoleImageTagMapOverride - description: | - Override the mapping of environmentd versions to console versions - default: | - {} - - - parameter: operator.features.createBalancers - description: | - Flag to indicate whether to create balancerd pods for the environments - default: true - - - parameter: operator.features.createConsole - description: | - Flag to indicate whether to create console pods for the environments - default: true + default: "25cc" - parameter: operator.image.pullPolicy description: | Policy for pulling the image: "IfNotPresent" avoids unnecessary re-pulling of images - default: IfNotPresent + default: "IfNotPresent" - parameter: operator.image.repository description: | The Docker repository for the operator image - default: materialize/orchestratord + default: "materialize/orchestratord" - parameter: operator.image.tag description: | The tag/version of the operator image to be used - default: v0.127.0 + default: "v0.138.0" - parameter: operator.nodeSelector description: "" - default: | - {} + default: "{}" - parameter: operator.resources.limits description: | Resource limits for the operator's CPU and memory - default: | - {"memory":"512Mi"} + default: '{"memory":"512Mi"}' - parameter: operator.resources.requests description: | Resources requested by the operator for CPU and memory - default: | - {"cpu":"100m","memory":"512Mi"} + default: '{"cpu":"100m","memory":"512Mi"}' + + - parameter: operator.secretsController + description: | + Which secrets controller to use for storing secrets. Valid values are 'kubernetes' and 'aws-secrets-manager'. + Setting 'aws-secrets-manager' requires a configured AWS cloud provider and IAM role for the environment with Secrets Manager permissions. + default: "kubernetes" - parameter: rbac.create description: | Whether to create necessary RBAC roles and bindings - default: true + default: "true" + + - parameter: schedulerName + description: | + Optionally use a non-default kubernetes scheduler. + default: "nil" - parameter: serviceAccount.create description: | Whether to create a new service account for the operator - default: true + default: "true" - parameter: serviceAccount.name description: | The name of the service account to be created - default: orchestratord + default: "orchestratord" - parameter: storage.storageClass.allowVolumeExpansion description: "" - default: false + default: "false" - parameter: storage.storageClass.create description: | - Set to false to use an existing StorageClass instead - default: false + Set to false to use an existing StorageClass instead. + Refer to the [Kubernetes StorageClass documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/) + default: "false" - parameter: storage.storageClass.name description: | - Name of the StorageClass to create/use: e.g.,"openebs-lvm-instance-store-ext4" + Name of the StorageClass to create/use: eg "openebs-lvm-instance-store-ext4" default: "" - parameter: storage.storageClass.parameters - description: Parameters for the CSI driver - default: | - {"fsType":"ext4","storage":"lvm","volgroup":"instance-store-vg"} + description: | + Parameters for the CSI driver + default: '{"fsType":"ext4","storage":"lvm","volgroup":"instance-store-vg"}' - parameter: storage.storageClass.provisioner description: | @@ -202,25 +234,24 @@ parameters: - parameter: storage.storageClass.reclaimPolicy description: "" - default: Delete + default: "Delete" - parameter: storage.storageClass.volumeBindingMode description: "" - default: WaitForFirstConsumer + default: "WaitForFirstConsumer" - parameter: telemetry.enabled description: "" - default: true + default: "true" - parameter: telemetry.segmentApiKey description: "" - default: hMWi3sZ17KFMjn2sPWo9UJGpOQqiba4A + default: "hMWi3sZ17KFMjn2sPWo9UJGpOQqiba4A" - parameter: telemetry.segmentClientSide description: "" - default: true + default: "true" - parameter: tls.defaultCertificateSpecs description: "" - default: | - {} + default: "{}" diff --git a/doc/user/data/self_managed/self_managed_cluster_sizes.yml b/doc/user/data/self_managed/self_managed_cluster_sizes.yml new file mode 100644 index 0000000000000..ed5b74582ea31 --- /dev/null +++ b/doc/user/data/self_managed/self_managed_cluster_sizes.yml @@ -0,0 +1,97 @@ +cluster_sizes: + - size: 25cc + workers: 1 + scale: 1 + cpu_exclusive: false + cpu_limit: 0.5 + credits_per_hour: "0.25" + disk_limit: "7762MiB" + memory_limit: "3881MiB" + - size: 50cc + workers: 1 + scale: 1 + cpu_exclusive: true + cpu_limit: 1 + credits_per_hour: "0.5" + disk_limit: "15525MiB" + memory_limit: "7762MiB" + - size: 100cc + workers: 2 + scale: 1 + cpu_exclusive: true + cpu_limit: 2 + credits_per_hour: "1" + disk_limit: "31050MiB" + memory_limit: "15525MiB" + - size: 200cc + workers: 4 + scale: 1 + cpu_exclusive: true + cpu_limit: 4 + credits_per_hour: "2" + disk_limit: "62100MiB" + memory_limit: "31050MiB" + - size: 300cc + workers: 6 + scale: 1 + cpu_exclusive: true + cpu_limit: 6 + credits_per_hour: "3" + disk_limit: "93150MiB" + memory_limit: "46575MiB" + - size: 400cc + workers: 8 + scale: 1 + cpu_exclusive: true + cpu_limit: 8 + credits_per_hour: "4" + disk_limit: "124201MiB" + memory_limit: "62100MiB" + - size: 600cc + workers: 12 + scale: 1 + cpu_exclusive: true + cpu_limit: 12 + credits_per_hour: "6" + disk_limit: "186301MiB" + memory_limit: "93150MiB" + - size: 800cc + workers: 16 + scale: 1 + cpu_exclusive: true + cpu_limit: 16 + credits_per_hour: "8" + disk_limit: "248402MiB" + memory_limit: "124201MiB" + - size: 1200cc + workers: 24 + scale: 1 + cpu_exclusive: true + cpu_limit: 24 + credits_per_hour: "12" + disk_limit: "372603MiB" + memory_limit: "186301MiB" + - size: 1600cc + workers: 31 + scale: 1 + cpu_exclusive: true + cpu_limit: 31 + credits_per_hour: "16" + disk_limit: "481280MiB" + memory_limit: "240640MiB" + - size: 3200cc + workers: 62 + scale: 1 + cpu_exclusive: true + cpu_limit: 62 + credits_per_hour: "32" + disk_limit: "962560MiB" + memory_limit: "481280MiB" + - size: 6400cc + workers: 62 + scale: 2 + cpu_exclusive: true + cpu_limit: 62 + credits_per_hour: "64" + disk_limit: "962560MiB" + memory_limit: "481280MiB" diff --git a/doc/user/data/self_managed/self_managed_operator_compatibility.yml b/doc/user/data/self_managed/self_managed_operator_compatibility.yml new file mode 100644 index 0000000000000..61152b2aacc29 --- /dev/null +++ b/doc/user/data/self_managed/self_managed_operator_compatibility.yml @@ -0,0 +1,67 @@ +columns: + - column: "Materialize Operator" + - column: "orchestratord version" + - column: "environmentd version" + - column: "Release date" + - column: "Notes" + +rows: + - "Materialize Operator": "v25.2.11" + "orchestratord version": "v0.159.1" + "environmentd version": "v0.147.17" + "Release date": "2025-10-14" + "Notes": "Fixes issue where licenceKeyChecks were enabled by default" + + - "Materialize Operator": "v25.2.10" + "orchestratord version": "v0.159.1" + "environmentd version": "v0.147.17" + "Release date": "2025-10-9" + "Notes": "Adds support for GCP using `COPY TO s3`, resolves console bug involving sources in object explorer, fixes 0dt cluster reconfiguration issue." + + - "Materialize Operator": "v25.2.9" + "orchestratord version": "v0.159.0" + "environmentd version": "v0.147.17" + "Release date": "2025-10-8" + "Notes": "Please use 25.2.10" + + - "Materialize Operator": "v25.2.5" + "orchestratord version": "v0.154.0." + "environmentd version": "v0.147.6" + "Release date": "2025-08-21" + "Notes": "Fixes license key issues, adds broader service account support." + + - "Materialize Operator": "v25.2.4" + "orchestratord version": "v0.153.0" + "environmentd version": "v0.147.4" + "Release date": "2025-07-15" + "Notes": "DO NOT USE" + + - "Materialize Operator": "v25.2.3" + "orchestratord version": "v0.147.5" + "environmentd version": "v0.147.4" + "Release date": "2025-07-15" + "Notes": "" + + - "Materialize Operator": "v25.2.0" + "orchestratord version": "v0.147.0" + "environmentd version": "v0.147.0" + "Release date": "2025-06-24" + "Notes": "Support for password authentication." + + - "Materialize Operator": "v25.1.12" + "orchestratord version": "v0.144.0" + "environmentd version": "v0.130.13" + "Release date": "2025-05-15" + "Notes": "" + + - "Materialize Operator": "v25.1.11" + "orchestratord version": "v0.143.0" + "environmentd version": "v0.130.12" + "Release date": "2025-05-15" + "Notes": "" + + - "Materialize Operator": "v25.1.10" + "orchestratord version": "v0.142.1" + "environmentd version": "v0.130.11" + "Release date": "2025-04-24" + "Notes": "" diff --git a/doc/user/data/self_managed/terraform_helm_compatibility.yml b/doc/user/data/self_managed/terraform_helm_compatibility.yml new file mode 100644 index 0000000000000..3efb494746b3d --- /dev/null +++ b/doc/user/data/self_managed/terraform_helm_compatibility.yml @@ -0,0 +1,53 @@ +columns: + - column: "terraform-helm-materialize" + - column: "Notes" + - column: "Release date" + +rows: + - "terraform-helm-materialize": "v0.1.26" + "Notes": | + - Uses as default Materialize Operator version: `v0.159.1` + - Uses as default environmentd version: `v0.147.17` + "Release date": "2025-10-14" + + - "terraform-helm-materialize": "v0.1.25" + "Notes": | + - Uses as default Materialize Operator version: `v0.159.1` + - Uses as default environmentd version: `v0.147.17` + "Release date": "2025-9-10" + + - "terraform-helm-materialize": "v0.1.16" + "Notes": | + - Adds support for passing in additional `environmentd` configuration options. + - Adds support for password authentication. + - Uses as default Materialize Operator version: `v25.2.0` + - Uses as default environmentd version: `v0.147.0` + "Release date": "2025-05-15" + + - "terraform-helm-materialize": "v0.1.15" + "Notes": | + - Adds support for passing in additional `environmentd` configuration options. + - Uses as default Materialize Operator version: `v25.1.12` + - Uses as default environmentd version: `v0.130.13` + "Release date": "2025-05-15" + + - "terraform-helm-materialize": "v0.1.14" + "Notes": | + + - Uses as default Materialize Operator version: `v25.1.12` + - Uses as default environmentd version: `v0.130.13` + "Release date": "2025-05-15" + + - "terraform-helm-materialize": "v0.1.13" + "Notes": | + + - Uses as default Materialize Operator version: `v25.1.11` + - Uses as default environmentd version: `v0.130.12` + "Release date": "2025-05-15" + + - "terraform-helm-materialize": "v0.1.12" + "Notes": | + + - Uses as default Materialize Operator version: `v25.1.7` + - Uses as default environmentd version: `v0.130.8` + "Release date": "2025-04-08" diff --git a/doc/user/data/self_managed/terraform_list.yml b/doc/user/data/self_managed/terraform_list.yml new file mode 100644 index 0000000000000..167e7215c9f8f --- /dev/null +++ b/doc/user/data/self_managed/terraform_list.yml @@ -0,0 +1,29 @@ +columns: + - column: "Sample Module" + - column: "Description" + +rows: +- "Sample Module": | + [terraform-helm-materialize](https://github.com/MaterializeInc/terraform-helm-materialize) + "Description": | + A sample Terraform module for installing the Materialize Helm chart into a Kubernetes cluster. + +- "Sample Module": | + [Materialize on AWS](https://github.com/MaterializeInc/terraform-aws-materialize) + "Description": | + A sample Terraform module for deploying Materialize on AWS Cloud Platform + with all required infrastructure components. + See [Install on AWS](/self-managed/installation/install-on-aws/) for an example usage. + +- "Sample Module": | + [Materialize on Azure](https://github.com/MaterializeInc/terraform-azurerm-materialize) + "Description": | + A sample Terraform module for deploying Materialize on Azure with all + required infrastructure components. See [Install on + Azure](/self-managed/installation/install-on-azure/) for an example usage. + +- "Sample Module": | + [Materialize on Google Cloud Platform (GCP)](https://github.com/MaterializeInc/terraform-google-materialize) + "Description": | + A sample Terraform module for deploying Materialize on Google Cloud Platform + (GCP) with all required infrastructure components. See [Install on GCP](/self-managed/installation/install-on-gcp/) for an example usage. diff --git a/doc/user/layouts/partials/head.html b/doc/user/layouts/partials/head.html index ec8c025927f1e..376a163a68307 100644 --- a/doc/user/layouts/partials/head.html +++ b/doc/user/layouts/partials/head.html @@ -121,6 +121,8 @@ {{/* Tabs */}} diff --git a/doc/user/layouts/partials/sidebar.html b/doc/user/layouts/partials/sidebar.html index f129b390ba7da..030185eb3fcef 100644 --- a/doc/user/layouts/partials/sidebar.html +++ b/doc/user/layouts/partials/sidebar.html @@ -11,10 +11,10 @@ {{ range .Site.Menus.main.ByWeight }}
  • + {{.Name | markdownify}} {{- if .HasChildren }} {{ readFile "static/images/icon_sidebar_expander.svg" | safeHTML }} {{ end }} - {{.Name | markdownify}} {{- if .HasChildren }} @@ -22,10 +22,10 @@ {{- range .Children }}
  • + {{.Name | markdownify}} {{- if .HasChildren }} {{ readFile "static/images/icon_sidebar_expander.svg" | safeHTML }} {{ end }} - {{.Name | markdownify}} {{- if .HasChildren }} @@ -33,10 +33,10 @@ {{- range .Children }}
  • + {{.Name | markdownify}} {{- if .HasChildren }} {{ readFile "static/images/icon_sidebar_expander.svg" | safeHTML }} {{ end }} - {{.Name | markdownify}} {{- if .HasChildren }} diff --git a/doc/user/layouts/partials/toc.html b/doc/user/layouts/partials/toc.html index 88bd1d17020cd..5cc5045764ba3 100644 --- a/doc/user/layouts/partials/toc.html +++ b/doc/user/layouts/partials/toc.html @@ -44,53 +44,72 @@

    On this page