diff --git a/.github/workflows/generate-helm-spec.yml b/.github/workflows/generate-helm-spec.yml new file mode 100644 index 00000000..ee0fa80f --- /dev/null +++ b/.github/workflows/generate-helm-spec.yml @@ -0,0 +1,93 @@ +--- + name: Generate Helm Spec Docs + on: + repository_dispatch: # Allows other repositories to trigger this workflow + types: [generate-helm-spec-docs] + workflow_dispatch: # Allows manual trigger of the workflow + jobs: + trigger: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.RP_AWS_CRED_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }} + - uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + ,sdlc/prod/github/actions_bot_token + parse-json-secrets: true + # Download the helm-docs tool which generates documentation from Helm charts. + - name: Download helm-docs + run: | + curl -sL https://github.com/norwoodj/helm-docs/releases/download/v1.14.2/helm-docs_1.14.2_Linux_x86_64.tar.gz | tar xz + # Add the helm-docs tool to the system's PATH for easy access. + - name: Add helm-docs to $PATH + run: | + echo "helm-docs" >> $GITHUB_PATH + sudo mv helm-docs /usr/local/bin/ + # Checkout the main branch of the helm-charts repository. + - name: Checkout helm-charts repository + uses: actions/checkout@v4 + with: + repository: redpanda-data/helm-charts + ref: main + path: helm-charts + token: ${{ env.ACTIONS_BOT_TOKEN }} + # Generate the Helm documentation using the helm-docs tool. + - name: Generate Helm docs + run: | + helm-docs + working-directory: ./helm-charts + # Install pandoc, a tool to convert between different markup formats. + - name: Install pandoc + run: | + sudo apt-get install pandoc + # Checkout the main branch of the redpanda-docs repository. + - name: Checkout documentation repository + uses: actions/checkout@v4 + with: + repository: redpanda-data/rp-connect-docs + ref: main + path: redpanda-docs + token: ${{ env.ACTIONS_BOT_TOKEN }} + # Convert the generated Markdown Helm documentation to AsciiDoc format using pandoc. + - name: Convert Markdown to AsciiDoc + run: | + pandoc ./helm-charts/charts/connect/README.md -t asciidoc -o ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + - name: Modify third-level headings format + run: | + sed -i 's/\(\[[0-9]*\)\]\./\1\\]\./g' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + sed -i 's/=== \(http\([^[]\|\%5[BbDd]\)*\)\[\([^]]*\)\]/=== link:++\1++\[\3\]/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + - name: Correct AsciiDoc format + run: | + # Correct the title format + sed -i 's/^== # \(.*\)/= \1/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + # Correct the description format + sed -i 's/^== description: \(.*\)/:description: \1/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + # Check if any changes were made in the documentation. + - name: Check if changes were made + id: check_changes + run: | + cd ./redpanda-docs + changes=$(git status --porcelain) + if [ -z "$changes" ]; then + echo "has_changes=false" >> $GITHUB_ENV + else + echo "has_changes=true" >> $GITHUB_ENV + fi + # If changes were detected, commit those changes. + - name: Commit changes + if: env.has_changes == 'true' + run: | + cd ./redpanda-docs + git config --global user.email "vbotbuildovich@users.noreply.github.com" + git config --global user.name "vbotbuildovich" + git add modules/reference/* + git commit -m "auto-docs: Update Helm spec" + git push origin main + env: + ACCESS_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }} \ No newline at end of file diff --git a/antora.yml b/antora.yml index 8a74c7f9..746d301b 100644 --- a/antora.yml +++ b/antora.yml @@ -1,7 +1,7 @@ name: redpanda-connect title: Connect version: ~ -start_page: ROOT:about.adoc +start_page: get-started:about.adoc nav: - modules/ROOT/nav.adoc asciidoc: diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index f3ddc802..9e89777e 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -1,6 +1,6 @@ site: title: Redpanda Docs - start_page: redpanda-connect:ROOT:about.adoc + start_page: redpanda-connect:get-started:about.adoc url: http://localhost:5002 robots: disallow urls: diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 8fd72e8b..a5222f90 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -1,5 +1,10 @@ -* xref:about.adoc[] -** xref:whats_new_rpcn.adoc[] + +* xref:get-started:index.adoc[Get Started] +** xref:get-started:about.adoc[] +** xref:get-started:whats-new.adoc[What's New] +** xref:get-started:quickstarts/index.adoc[Quickstarts] +*** xref:get-started:quickstarts/rpk.adoc[CLI] +*** xref:get-started:quickstarts/helm-chart.adoc[Helm Chart] * xref:configuration:about.adoc[] ** xref:configuration:resources.adoc[] @@ -295,7 +300,6 @@ ** xref:components:logger/about.adoc[] * xref:guides:index.adoc[] -** xref:guides:getting_started.adoc[] ** xref:guides:bloblang/about.adoc[] *** xref:guides:bloblang/walkthrough.adoc[] *** xref:guides:bloblang/functions.adoc[] @@ -318,6 +322,7 @@ *** xref:guides:migration/v4.adoc[] *** xref:guides:migration/v3.adoc[] *** xref:guides:migration/v2.adoc[] + * xref:cookbooks:index.adoc[] ** xref:cookbooks:custom_metrics.adoc[] ** xref:cookbooks:discord_bot.adoc[] @@ -326,3 +331,6 @@ ** xref:cookbooks:joining_streams.adoc[] ** xref:cookbooks:rag.adoc[] ** xref:cookbooks:redpanda_migrator.adoc[] + +* xref:reference:index.adoc[] +** xref:reference:k-connect-helm-spec.adoc[Helm Chart Specification] diff --git a/modules/ROOT/pages/about.adoc b/modules/get-started/pages/about.adoc similarity index 93% rename from modules/ROOT/pages/about.adoc rename to modules/get-started/pages/about.adoc index 6e2e251e..e09ad07b 100644 --- a/modules/ROOT/pages/about.adoc +++ b/modules/get-started/pages/about.adoc @@ -1,5 +1,5 @@ = Introduction to Redpanda Connect -:page-aliases: guides:delivery_guarantee.adoc +:page-aliases: guides:delivery_guarantee.adoc, ROOT:about.adoc Redpanda Connect is a declarative data streaming service that solves a wide range of data engineering problems with simple, chained, stateless xref:components:processors/about.adoc[processing steps]. It implements transaction-based resiliency with back pressure, so when connecting to at-least-once sources and sinks it's able to guarantee at-least-once delivery without needing to persist messages during transit. diff --git a/modules/get-started/pages/index.adoc b/modules/get-started/pages/index.adoc new file mode 100644 index 00000000..11859c48 --- /dev/null +++ b/modules/get-started/pages/index.adoc @@ -0,0 +1,5 @@ += Get Started +:description: Explore these topics to get started with Redpanda Connect. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc new file mode 100644 index 00000000..03920d4b --- /dev/null +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -0,0 +1,528 @@ += Get Started with the Redpanda Connect Helm Chart +:description: Deploy Redpanda Connect on Kubernetes using Helm. +:page-aliases: get-started:helm-chart.adoc + +This guide explains how to deploy and configure Redpanda Connect on Kubernetes using the Helm chart. It covers the available deployment modes and walks you through building your own pipelines. + +== What is Redpanda Connect? + +Redpanda Connect is a powerful stream processor that integrates data across various sources (inputs) and sinks (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it an ideal solution for data pipelines that require high performance and resilience. + +Redpanda Connect includes Bloblang, a flexible mapping language for processing data with built-in functions for transformations, random data generation, and more. These functions allow for easy customization of data pipelines. + +Common use cases for Redpanda Connect include: + +* **Real-time data ingestion**: Ingesting data from various sources into Redpanda or other data platforms. +* **Data transformation**: Enriching and transforming data to match business requirements before forwarding it to a destination. +* **Data filtering and routing**: Filtering data and routing it to the appropriate sinks based on predefined conditions. + +== Prerequisites + +* A running Kubernetes cluster. +* The https://kubernetes.io/docs/tasks/tools/[`kubectl` CLI] and the https://helm.sh/docs/intro/install/[`helm` CLI] installed. + +== Install with Helm + +To deploy Redpanda Connect on Kubernetes, use the official Helm chart. + +[source,bash] +---- +helm repo add redpanda https://charts.redpanda.com <1> +helm repo update <2> +helm install redpanda-connect redpanda/connect --namespace --create-namespace <3> +---- + +<1> Adds the Redpanda Helm repository. +<2> Updates your local Helm repository cache. +<3> Installs Redpanda Connect in the given namespace. You can customize this deployment by configuring values in the Helm chart. + +== Deployment modes + +Redpanda Connect can run in two different modes: **standalone mode** and **streams mode**. + +* **Standalone mode**: Allows you to run a single pipeline at a time, making it suitable for simpler configurations that don't need to run concurrently. +* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing use cases. + +== Run pipelines in standalone mode + +In standalone mode, you can configure a single pipeline within the `config` section of your Helm values file. + +=== Hello world + +In this example, you'll produce a simple message and convert it to uppercase using a Bloblang method. + +. Create a `pipeline.yaml` file. ++ +This file will be used to override the default Helm chart values for Redpanda Connect. + +. Add the following configuration to `pipeline.yaml`: ++ +[source,yaml] +---- +config: + input: + generate: + mapping: | + root.message = "Hello, Redpanda Connect!" <1> + count: 1 <2> + pipeline: + processors: + - mapping: | + root.message = this.message.uppercase() <3> + output: + stdout: {} <4> +---- +<1> Use Bloblang to create a JSON object with the message "Hello, Redpanda Connect!" stored in `root.message`. +<2> Generate the message only once. +<3> Convert the `message` field in the input to uppercase using the xref:guides:bloblang/methods.adoc#uppercase[`uppercase()`] method in Bloblang. +<4> Output the processed data to stdout, making it viewable in logs. + +. Deploy the pipeline: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Check the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace +---- ++ +You should see the message converted to uppercase in the output: ++ +[,json] +---- +{"message":"HELLO, REDPANDA CONNECT!"} +---- + +. Check the Pod's status: ++ +[source,bash] +---- +kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +---- ++ +The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the pipeline, the Pod exits, causing Kubernetes to restart it repeatedly. ++ +To prevent this status, you can configure Redpanda Connect to continue processing data indefinitely. + +=== Produce continuous data + +To produce data continuously, you can set `input.generate.count` to `0`. + +. Update the `pipeline.yaml` file to produce a message every second, indefinitely: ++ +[source,yaml] +---- +config: + input: + generate: + interval: 1s + count: 0 # Setting count to 0 ensures it generates data indefinitely. + mapping: | + root.message = "Hello, Redpanda Connect!" + pipeline: + processors: + - mapping: | + root.message = this.uppercase() + output: + stdout: {} +---- + +. Deploy the updated configuration: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see in the logs that Redpanda Connect is producing the same message every second and its being converted to uppercase: ++ +[source,json,role="no-copy"] +---- +{"message": "HELLO, REDPANDA CONNECT!"} +{"message": "HELLO, REDPANDA CONNECT!"} +{"message": "HELLO, REDPANDA CONNECT!"} +---- + +. Check the Pod's status: ++ +[source,bash] +---- +kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +---- ++ +The Pod should now be running without entering a `CrashLoopBackOff` state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. + +=== Simulate realistic data streams + +To make the output more realistic, use some Bloblang functions to generate varied data such as random names and emails. + +. Update the `pipeline.yaml` file to generate some realistic user data. ++ +[source,yaml] +---- +config: + input: + generate: + interval: 1s + count: 0 + mapping: | + # Store the generated names in variables + let first_name = fake("first_name") + let last_name = fake("last_name") + + # Build the message + root.user_id = counter() + root.name = ($first_name + " " + $last_name) + root.timestamp = now() + pipeline: + processors: + - mapping: | + root.name = this.name.uppercase() + output: + stdout: {} +---- ++ +This configuration generates a JSON object with: ++ +- `user_id`: A unique identifier for each record, generated using the xref:guides:bloblang/functions.adoc#counter[`counter()`] function. +- `name`: A randomly generated first and last name, using the xref:guides:bloblang/functions.adoc#fake[`fake()`] function. The first and last names are stored in variables and referenced using the `$` syntax. +- `timestamp`: The current timestamp at the time of generation, using the xref:guides:bloblang/functions.adoc#now[`now()`] function. + +. Deploy the updated configuration: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see logs showing JSON objects similar to the following, with names in uppercase: ++ +[source,json,role="no-copy"] +---- +{"name":"ZOIE SIPES"} +{"name":"LORENA KERTZMANN"} +{"name":"DALLAS BOYER"} +{"name":"LOUIE WILDERMAN"} +{"name":"EMILIA KOEPP"} +{"name":"KALEIGH PACOCHA"} +---- + +=== Process data from a file input + +To configure a pipeline that reads data from a file, first store the data in a ConfigMap. This ConfigMap will be mounted into the Redpanda Connect Pod, allowing it to read the file directly. + +. Create a ConfigMap to provide the input data that Redpanda Connect will read. This example ConfigMap contains a JSON object with example user data: ++ +[source,bash] +---- +kubectl create configmap connect-input --from-literal=input-data='{"name": "Redpanda Connect", "email": "rp.connect@example.com"}' --namespace +---- ++ +This ConfigMap will act as the source for the file-based input in Redpanda Connect, allowing the pipeline to read and process this structured JSON data. + +. Update the `pipeline.yaml` file to read data from the file mounted by the ConfigMap: ++ +.`pipeline.yaml` +[source,yaml] +---- +extraVolumes: + - name: input-config + configMap: + name: connect-input +extraVolumeMounts: + - name: input-config + mountPath: /input <1> + subPath: input-data +config: + input: + file: + paths: + - "/input" <1> + pipeline: + processors: + - mapping: | + root.name = this.name.uppercase() + output: + stdout: {} +---- ++ +<1> For the input, use the contents of the file at the path where the ConfigMap data is mounted. + +. Deploy the pipeline: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Check the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace +---- ++ +You should see the username converted to uppercase in the output: ++ +[,json] +---- +{"name":"REDPANDA CONNECT"} +---- + +== Run multiple pipelines in streams mode + +In streams mode, each pipeline, defined in separate YAML files, runs simultaneously, making this mode ideal for high-throughput applications. All the YAML files must be bundled together into a ConfigMap that you can pass to Redpanda Connect. + +. Define your pipeline configurations in the following separate YAML files: ++ +.`woof.yaml` +[source,yaml] +---- +input: + generate: + mapping: root = "woof" # Generates a message with the word "woof" at regular intervals. + interval: 5s + count: 0 +output: + stdout: + codec: lines # Outputs each message as a new line in stdout. +---- ++ +.`meow.yaml` +[source,yaml] +---- +input: + generate: + mapping: root = "meow" # Generates a message with the word "meow" at regular intervals. + interval: 2s + count: 0 +output: + stdout: + codec: lines # Outputs each message as a new line in stdout. +---- + +. Bundle the configuration files into a ConfigMap, which Redpanda Connect will reference: ++ +[source,bash] +---- +kubectl create configmap connect-streams --from-file=woof.yaml --from-file=meow.yaml --namespace +---- + +. Configure Redpanda Connect in streams mode and specify the name of the ConfigMap to use: ++ +.`connect.yaml` +[source,yaml] +---- +streams: + enabled: true <1> + streamsConfigMap: "connect-streams" <2> +---- ++ +<1> Enable streams mode in Redpanda Connect. +<2> Use the given ConfigMap as the pipeline configuration. + +. Deploy the chart: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values connect.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see logs showing a combination of outputs from both pipelines: ++ +[.no-copy] +---- +woof +meow +meow +meow +woof +meow +meow +---- + +=== Update the pipeline in streams mode + +To update a pipeline in streams mode: + +. Modify one of the configuration files locally. ++ +.`woof.yaml` +[source,yaml] +---- +# Updated woof.yaml +input: + generate: + mapping: root = "bark" # Updated to generate a message with the word "bark" instead of "woof." + interval: 5s + count: 0 +output: + stdout: + codec: lines +---- + +. Update the ConfigMap with the modified file: ++ +[source,bash] +---- +kubectl create configmap connect-streams --from-file=woof.yaml --from-file=meow.yaml --namespace --dry-run=client -o yaml | kubectl apply -f - +---- + +. Restart the Deployment: ++ +[source,bash] +---- +kubectl rollout restart deployment/redpanda-connect --namespace +---- + +=== Global configuration + +When deploying Redpanda Connect in streams mode, you can configure global tracing, logging, and HTTP settings to apply across all pipelines. Specify these in your `values.yaml` overrides under the `metrics`, `logger`, and `tracing` sections. + +[source,yaml] +---- +metrics: + prometheus: {} # Enable Prometheus metrics collection. + +tracing: + openTelemetry: + http: [] # Configure OpenTelemetry HTTP tracing. + grpc: [] + tags: {} + +logger: + level: INFO # Set logging level (e.g., INFO, DEBUG). + static_fields: + '@service': redpanda-connect # Add static fields to logs for better traceability. +---- + +== Access the HTTP server on Redpanda Connect + +To manage and monitor Redpanda Connect, you can use its HTTP server, which provides useful endpoints for version checking, pipeline management, and more. By default, Redpanda Connect exposes this server using a Kubernetes ClusterIP Service, accessible only within the cluster. + +. Forward the ports of the ClusterIP Service to your local device: ++ +[source,bash] +---- +kubectl port-forward svc/redpanda-connect 8080:80 --namespace +---- + +. Access the HTTP server locally. For example, to check the Redpanda Connect version, run: ++ +[,bash] +---- +curl http://localhost:8080/version +---- ++ +Example output: ++ +[,json] +---- +{ + "version": "v4.38.0", + "built": "2024-10-17T09:27:42Z" +} +---- + +You can also configure external access using a LoadBalancer Service or an Ingress. See the link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml#L79C1-L107C31[Helm values] for more details. + +== Upgrade Redpanda Connect + +To upgrade Redpanda Connect to a newer chart version: + +. Ensure your Helm repository is updated to fetch the latest version of the Redpanda chart: ++ +[,bash] +---- +helm repo update +---- + +. Upgrade your existing deployment using the https://helm.sh/docs/helm/helm_upgrade/[`helm upgrade`] command: ++ +[,bash] +---- +helm upgrade redpanda-connect redpanda/connect --namespace --values +---- ++ +Replace `` with the path to your custom configuration file. + +=== Roll back an upgrade + +If an upgrade introduces issues, you can roll back to a previous release. Helm retains previous deployments, allowing you to roll back with the https://helm.sh/docs/helm/helm_rollback/[`helm rollback`] command: + +[,bash] +---- +helm rollback redpanda-connect --namespace +---- + +To find the revision number, use the https://helm.sh/docs/helm/helm_history/[`helm history`] command: + +[,bash] +---- +helm history redpanda-connect --namespace +---- + +This command lists previous deployments and their revision numbers. + +== Uninstall Redpanda Connect + +To remove Redpanda Connect and all related resources from your Kubernetes cluster, use the https://helm.sh/docs/helm/helm_uninstall/[`helm uninstall`] command to uninstall the chart: + +[,bash] +---- +helm uninstall redpanda-connect --namespace +---- + +This command deletes all resources created by the Helm chart, including Deployments and Services. + +Uninstalling the chart does not delete the ConfigMaps that you manually created outside of the Helm chart. To delete these ConfigMaps, do the following: + +[,bash] +---- +kubectl delete configmap connect-streams connect-input --namespace +---- + +== Next steps + +* Learn more about xref:guides:bloblang/walkthrough.adoc[Bloblang], the mapping language for processing data in Redpanda Connect. +* Try more hands-on examples with one of the xref:cookbooks:index.adoc[Cookbooks]. + +== Suggested reading + +* xref:guides:streams_mode/about.adoc[Streams mode] +* xref:components:inputs/about.adoc[Inputs] +* xref:components:processors/about.adoc[Processors] +* xref:components:outputs/about.adoc[Outputs] +* xref:components:http/about.adoc[HTTP server] +* link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[Helm values] diff --git a/modules/get-started/pages/quickstarts/index.adoc b/modules/get-started/pages/quickstarts/index.adoc new file mode 100644 index 00000000..8225d2c6 --- /dev/null +++ b/modules/get-started/pages/quickstarts/index.adoc @@ -0,0 +1,5 @@ += Quickstarts +:description: Use your preferred tool to spin up a local instance of Redpanda Connect and get hands-on experience building your first pipelines. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/guides/pages/getting_started.adoc b/modules/get-started/pages/quickstarts/rpk.adoc similarity index 89% rename from modules/guides/pages/getting_started.adoc rename to modules/get-started/pages/quickstarts/rpk.adoc index b600d98a..f14ec265 100644 --- a/modules/guides/pages/getting_started.adoc +++ b/modules/get-started/pages/quickstarts/rpk.adoc @@ -1,12 +1,12 @@ -= Get Started -:description: Getting started with Redpanda Connect. -:page-aliases: getting_started:overview.adoc, ROOT:install.adoc += Get Started with Redpanda Connect using `rpk` +:description: pass:q[Deploy your first pipelines using Redpanda Connect and `rpk`.] +:page-aliases: getting_started:overview.adoc, ROOT:install.adoc, guides:getting_started.adoc -This topic explains how to get started with Redpanda Connect. +This guide explains how to get started with Redpanda Connect using `rpk`, the Redpanda command-line interface (CLI). == Install -You can install Redpanda Connect using the `rpk` command-line tool (CLI). The `rpk` CLI allows you to create and manage data pipelines with Redpanda Connect as well as xref:ROOT:get-started:rpk-install.adoc[interact with Redpanda clusters]. +The `rpk` CLI allows you to create and manage data pipelines with Redpanda Connect as well as xref:ROOT:get-started:rpk-install.adoc[interact with Redpanda clusters]. [%collapsible] .Also interacting with a Redpanda cluster? @@ -31,7 +31,7 @@ To use `rpk` in Docker, you can use the Redpanda Docker image, which includes bo docker pull docker.redpanda.com/redpandadata/connect ---- -== Run +== Run a pipeline A Redpanda Connect stream pipeline is configured with a single xref:configuration:about.adoc[config file], you can generate a fresh one with: diff --git a/modules/ROOT/pages/whats_new_rpcn.adoc b/modules/get-started/pages/whats-new.adoc similarity index 94% rename from modules/ROOT/pages/whats_new_rpcn.adoc rename to modules/get-started/pages/whats-new.adoc index ab12f3da..33d9a18f 100644 --- a/modules/ROOT/pages/whats_new_rpcn.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -1,5 +1,6 @@ = What's New in Redpanda Connect :description: Summary of new features in Redpanda Connect. +:page-aliases: ROOT:whats_new_rpcn.adoc This topic includes new content added from version 4.29.0 onwards. @@ -9,6 +10,14 @@ For a full list of product updates, and details of updates prior to version 4.29 This update includes the beta release of two new outputs for Cloud and Self-Managed, and the introduction of a xref:configuration:telemetry.adoc[telemetry service], which collects and sends anonymized usage statistics to Redpanda. +=== New tooling + +A new Helm chart for deploying Redpanda Connect in Kubernetes is now available. This chart simplifies the deployment and configuration of Redpanda Connect, allowing you to manage and scale Redpanda Connect instances in Kubernetes environments. + +To get started with the Helm chart, see xref:get-started:quickstarts/helm-chart.adoc[]. + +Or, see the release notes for the chart on https://github.com/redpanda-data/helm-charts/releases/tag/connect-3.0.0[GitHub]. + === New outputs (beta) * xref:components:outputs/azure_data_lake_gen2.adoc[`azure_data_lake_gen2`] (certified) diff --git a/modules/reference/pages/index.adoc b/modules/reference/pages/index.adoc new file mode 100644 index 00000000..8e1a7f85 --- /dev/null +++ b/modules/reference/pages/index.adoc @@ -0,0 +1,3 @@ += Reference +:description: The following topics provide reference material for Redpanda Connect. +:page-layout: index \ No newline at end of file diff --git a/modules/reference/pages/k-connect-helm-spec.adoc b/modules/reference/pages/k-connect-helm-spec.adoc new file mode 100644 index 00000000..71a0b87f --- /dev/null +++ b/modules/reference/pages/k-connect-helm-spec.adoc @@ -0,0 +1,589 @@ += Redpanda Connect Chart Specification +:description: Find the default values and descriptions of settings in the Redpanda Connect Helm chart. + +image:https://img.shields.io/badge/Version-3.0.0-informational?style=flat-square[Version: +3.0.0] +image:https://img.shields.io/badge/Type-application-informational?style=flat-square[Type: +application] +image:https://img.shields.io/badge/AppVersion-4.38.0-informational?style=flat-square[AppVersion: +4.38.0] + +Redpanda Connect is a high performance and resilient stream processor, +able to connect various sources and sinks in a range of brokering +patterns and perform hydration, enrichments, transformations and filters +on payloads. + +This Helm chart deploys a Redpanda Connect instance in either streams +mode or standalone. + +This page describes the contents of the chart's +https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[`values.yaml` +file]. Each of the settings is listed and described on this page, along +with any default values. + +For instructions on how to install and use the chart, including how to +override and customize the chart’s values, refer to the +https://docs.redpanda.com/redpanda-connect/get-started/helm-chart/[deployment +documentation]. + +NOTE: To migrate to this chart from the +https://github.com/redpanda-data/redpanda-connect-helm-chart[Benthos +Helm chart], see the +https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/MIGRATION_FROM_BENTHOS.md[migration +guide in this GitHub repository]. + +''''' + +Autogenerated from chart metadata using +https://github.com/norwoodj/helm-docs/releases/v1.14.2[helm-docs +v1.14.2] + +== Source Code + +* https://github.com/redpanda-data/helm-charts + +== Settings + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=affinity[affinity] + +Affinity rules for scheduling Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=args[args] + +Override the default arguments for the entrypoint command. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling[autoscaling] + +Configuration for Horizontal Pod Autoscaling for the Redpanda Connect +Deployment. + +*Default:* + +.... +{"enabled":false,"maxReplicas":12,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.enabled[autoscaling.enabled] + +Enable Horizontal Pod Autoscaling. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.maxReplicas[autoscaling.maxReplicas] + +Maximum number of replicas if autoscaling is enabled. + +*Default:* `12` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.metrics[autoscaling.metrics] + +List of metrics to use for scaling, such as CPU utilization. + +*Default:* + +.... +[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}] +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.minReplicas[autoscaling.minReplicas] + +Minimum number of replicas if autoscaling is enabled. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=command[command] + +Override the default entrypoint command of the container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=commonLabels[commonLabels] + +Additional labels to apply to all resources created by this chart. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=config[config] + +Configuration for `/redpanda-connect.yaml` file. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.annotations[deployment.annotations] + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe[deployment.livenessProbe] + +Configuration for the liveness probe that checks if the container is +running. + +*Default:* + +.... +{"failureThreshold":3,"httpGet":{"path":"/ping","port":"http"},"periodSeconds":5,"successThreshold":1,"timeoutSeconds":2} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.failureThreshold[deployment.livenessProbe.failureThreshold] + +Number of consecutive failures before Pod is restarted. + +*Default:* `3` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.httpGet[deployment.livenessProbe.httpGet] + +HTTP GET request configuration for the liveness probe. + +*Default:* + +.... +{"path":"/ping","port":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.periodSeconds[deployment.livenessProbe.periodSeconds] + +Frequency (in seconds) of liveness checks. + +*Default:* `5` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.successThreshold[deployment.livenessProbe.successThreshold] + +Number of consecutive successes required to mark as healthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.timeoutSeconds[deployment.livenessProbe.timeoutSeconds] + +Timeout (in seconds) for each liveness check. + +*Default:* `2` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.podAnnotations[deployment.podAnnotations] + +Annotations to add to Redpanda Connect Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.podLabels[deployment.podLabels] + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe[deployment.readinessProbe] + +Configuration for the readiness probe that checks if the container is +ready to accept traffic. + +*Default:* + +.... +{"failureThreshold":1,"httpGet":{"path":"/ready","port":"http"},"periodSeconds":5,"successThreshold":1,"timeoutSeconds":2} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.failureThreshold[deployment.readinessProbe.failureThreshold] + +Number of consecutive failures before marking Pod as unhealthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.httpGet[deployment.readinessProbe.httpGet] + +HTTP GET request configuration for the readiness probe. + +*Default:* + +.... +{"path":"/ready","port":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.periodSeconds[deployment.readinessProbe.periodSeconds] + +Frequency (in seconds) of readiness checks. + +*Default:* `5` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.successThreshold[deployment.readinessProbe.successThreshold] + +Number of consecutive successes before marking Pod as healthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.timeoutSeconds[deployment.readinessProbe.timeoutSeconds] + +Timeout (in seconds) for each readiness check. + +*Default:* `2` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.replicaCount[deployment.replicaCount] + +Number of Replicas for the Redpanda Connect Deployment. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.restartPolicy[deployment.restartPolicy] + +Restart policy for containers in the Pod. + +*Default:* `"Always"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.rolloutConfigMap[deployment.rolloutConfigMap] + +Rollout the deployment on ConfigMap changes. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.terminationGracePeriodSeconds[deployment.terminationGracePeriodSeconds] + +Time in seconds for Pod termination grace period. + +*Default:* `60` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=env[env] + +Environment variables to pass to the Redpanda Connect container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=envFrom[envFrom] + +Names of Secrets or ConfigMaps that contain environment variables to +pass to the Redpanda Connect container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=extraVolumeMounts[extraVolumeMounts] + +Additional volume mounts for the container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=extraVolumes[extraVolumes] + +Additional volumes to add to the Pod. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=fullnameOverride[fullnameOverride] + +Override for the full name template. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http[http] + +Configuration for the HTTP server used to expose Redpanda Connect +endpoints. + +*Default:* + +.... +{"address":"0.0.0.0:4195","cors":{"enabled":false},"debug_endpoints":false,"enabled":true,"root_path":"/redpanda-connect"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.address[http.address] + +Bind Address for the HTTP server. + +*Default:* `"0.0.0.0:4195"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.cors.enabled[http.cors.enabled] + +Enable CORS support for HTTP server. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.debug_endpoints[http.debug_endpoints] + +Enable debug endpoints for the HTTP server. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.enabled[http.enabled] + +Enable HTTP server. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.root_path[http.root_path] + +Root path for HTTP API endpoints. + +*Default:* `"/redpanda-connect"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image[image] + +Configuration for the Redpanda Connect container image. + +*Default:* + +.... +{"pullPolicy":"IfNotPresent","repository":"docker.redpanda.com/redpandadata/connect","tag":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.pullPolicy[image.pullPolicy] + +Image pull policy, such as `IfNotPresent` or `Always`. + +*Default:* `"IfNotPresent"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.repository[image.repository] + +Docker repository for the Redpanda Connect image. + +*Default:* + +.... +"docker.redpanda.com/redpandadata/connect" +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.tag[image.tag] + +Override for the default image tag (uses chart `appVersion` if empty). + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=imagePullSecrets[imagePullSecrets] + +List of Secrets for private Docker registry authentication. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.annotations[ingress.annotations] + +Annotations for the Ingress resource. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.className[ingress.className] + +Ingress Class name for the Ingress resource. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.enabled[ingress.enabled] + +Enable Ingress resource creation. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.hosts[ingress.hosts] + +List of host definitions for the Ingress resource. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.tls[ingress.tls] + +TLS configuration for Ingress resource. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=initContainers[initContainers] + +Additional init containers to add to the Deployment. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=nameOverride[nameOverride] + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=nodeSelector[nodeSelector] + +Node selector for scheduling Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podDisruptionBudget[podDisruptionBudget] + +Configuration for the PodDisruptionBudget, which controls voluntary +evictions of Redpanda Connect Pods. + +*Default:* `{"enabled":false}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podDisruptionBudget.enabled[podDisruptionBudget.enabled] + +Enable a PodDisruptionBudget for the Deployment. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podSecurityContext[podSecurityContext] + +Security context to be applied at the Pod level. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=resources[resources] + +Resource limits and requests for the container. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=securityContext[securityContext] + +Security context to be applied to containers in the Pod. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service[service] + +Configuration for the Kubernetes Service that exposes Redpanda Connect. + +*Default:* + +.... +{"extraPorts":null,"name":"http","port":80,"protocol":"TCP","targetPort":"http","type":"ClusterIP"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.extraPorts[service.extraPorts] + +Additional ports to expose through the Service. + +*Default:* `nil` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.name[service.name] + +Name of the Kubernetes Service. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.port[service.port] + +Service port for accessing the application. + +*Default:* `80` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.protocol[service.protocol] + +Protocol to use for the Service, for example TCP or UDP. + +*Default:* `"TCP"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.targetPort[service.targetPort] + +Target port on the container. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.type[service.type] + +Type of Kubernetes Service, for example ClusterIP, NodePort, or +LoadBalancer. + +*Default:* `"ClusterIP"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount[serviceAccount] + +Configuration for the Kubernetes ServiceAccount associated with the +Redpanda Connect Pods. + +*Default:* + +.... +{"annotations":{},"create":true,"name":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.annotations[serviceAccount.annotations] + +Annotations to add to the ServiceAccount. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.create[serviceAccount.create] + +Specify whether a ServiceAccount should be created. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.name[serviceAccount.name] + +Name of the ServiceAccount to use. If not set and `create` is `true`, a +name is autogenerated. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor[serviceMonitor] + +Configuration for the ServiceMonitor used by Prometheus to scrape +Redpanda Connect metrics. + +*Default:* + +.... +{"enabled":false,"interval":"10s","scheme":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.enabled[serviceMonitor.enabled] + +Enable the creation of a ServiceMonitor for Prometheus monitoring. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.interval[serviceMonitor.interval] + +Scrape interval for Prometheus monitoring. + +*Default:* `"10s"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.scheme[serviceMonitor.scheme] + +HTTP Scheme to use for scraping. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams[streams] + +Configuration settings for the streams mode feature in Redpanda Connect. + +*Default:* + +.... +{"api":{"enable":true},"enabled":false,"streamsConfigMap":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.api.enable[streams.api.enable] + +Enable the streams API. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.enabled[streams.enabled] + +Enable streams mode. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.streamsConfigMap[streams.streamsConfigMap] + +Name of the ConfigMap that contains stream configuration files. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=tolerations[tolerations] + +Tolerations for scheduling Pods onto nodes with matching taints. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=topologySpreadConstraints[topologySpreadConstraints] + +Spread incoming Pods across the cluster based on topology to improve distribution and fault tolerance. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=updateStrategy[updateStrategy] + +Update strategy for the Deployment, for example `RollingUpdate`. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=watch[watch] + +Experimental feature: Watch config files for changes and automatically +apply them without needing to rollout the Deployment. Make sure that +`deployment.rolloutConfigMap` is `false`. + +*Default:* `false`