demo updates

README.md

@@ -149,11 +149,12 @@ redis-connect\bin> redisconnect.bat start
 
 <table>
     <tr>
-        <td><a href="/examples/db2/demo/README.md"><img src="/images/icons/DB2 Icon.png" style="float: center;" width="125" height="100"/></a></td> 
-        <td><a href="/examples/mysql/demo/README.md"><img src="/images/icons/MySQL Icon.png" style="float: center;" width="125" height="100"/></a></td> 
-        <td><a href="/examples/oracle/demo/README.md"><img src="/images/icons/Oracle Icon.png" style="float: center;" width="125" height="100"/></a></td> 
-        <td><a href="/examples/postgres/demo/README.md"><img src="/images/icons/Postgres Icon.png" style="float: center;" width="125" height="100"/></a></td> 
-        <td><a href="/examples/mssql/demo/README.md"><img src="/images/icons/SQL Server Icon.png" style="float: center;" width="125" height="100"/></a></td> 
+        <td><a href="/examples/db2/demo/README.md"><img src="/images/icons/DB2 Icon.png" style="float: center;" width="125" height="100"/></a></td>
+        <td><a href="/examples/mongodb/demo/README.md"><img src="/images/icons/MongoDB Icon.png" style="float: center;" width="125" height="100"/></a></td>
+        <td><a href="/examples/mysql/demo/README.md"><img src="/images/icons/MySQL Icon.png" style="float: center;" width="125" height="100"/></a></td>
+        <td><a href="/examples/oracle/demo/README.md"><img src="/images/icons/Oracle Icon.png" style="float: center;" width="125" height="100"/></a></td>
+        <td><a href="/examples/postgres/demo/README.md"><img src="/images/icons/Postgres Icon.png" style="float: center;" width="125" height="100"/></a></td>
+        <td><a href="/examples/mssql/demo/README.md"><img src="/images/icons/SQL Server Icon.png" style="float: center;" width="125" height="100"/></a></td>
         <td><a href="/examples/vertica/demo/README.md"><img src="/images/icons/Vertica Icon.png" style="float: center;" width="125" height="100"/></a></td>
     </tr>
 </table>

examples/db2/demo/config/jobmanager.properties

@@ -16,7 +16,7 @@
 #job.claim.attempt.interval=5000
 #job.claim.batch.size.per.attempt=4
 #job.claim.max.capacity=4
-#job.claim.heartbeat.lease.renewal.interval=5000
+#job.claim.heartbeat.lease.renewal.interval=10000
 
 #####     REST properties
 #rest.api.enabled=true

examples/mongodb/README.md

@@ -0,0 +1,12 @@
+## Setting up MongoDB (Source)
+
+Please see <a href="https://debezium.io/documentation/reference/stable/connectors/mongodb.html#setting-up-mongodb" target="_blank">MongoDB Setup</a> for reference.
+
+## Setting up Redis Enterprise Databases (Target)
+
+Before using Redis Connect to capture the changes committed on SQL Server into Redis Enterprise Database, first create a database for the metadata management and metrics provided by Redis Connect by creating a database with [RedisTimeSeries](https://redislabs.com/modules/redis-timeseries/) module enabled, see [Create Redis Enterprise Database](https://docs.redislabs.com/latest/rs/administering/creating-databases/#creating-a-new-redis-database) for reference. Then, create (or use an existing) another Redis Enterprise database (Target) to store the changes coming from SQL Server. Additionally, you can enable [RediSearch 2.0](https://redislabs.com/blog/introducing-redisearch-2-0/) module on the target database to enable secondary index with full-text search capabilities on the existing hashes where SQL Server changed events are being written at then [create an index, and start querying](https://oss.redislabs.com/redisearch/Commands/) the document in hashes.
+
+| ℹ️                                          |
+|:--------------------------------------------|
+| Docker demo: Follow the [Docker demo](demo) |
+| K8s Setup: Follow the [k8s-docs](k8s-docs)  |

examples/mongodb/demo/README.md

@@ -0,0 +1,179 @@
+# Demo Outline
+:white_check_mark: Setup MongoDB database (Source)<br>
+:white_check_mark: Setup and start Redis Enterprise database (Target)<br>
+:white_check_mark: Setup and start Redis Connect<br>
+:white_check_mark: Perform Initial load and CDC with Redis Connect<br>
+
+# Prerequisites
+Docker compatible [*nix OS](https://en.wikipedia.org/wiki/Unix-like) and [Docker](https://docs.docker.com/get-docker) installed.
+<br>Please have 8 vCPU*, 8GB RAM and 50GB storage for this demo to function properly. Adjust the resources based on your requirements. For HA, at least have 2 Redis Connect Connector instances deployed on separate hosts.</br>
+<br>Execute the following commands (copy & paste) to download and setup Redis Connect and demo scripts.
+i.e.</br>
+
+```bash
+wget -c https://github.com/redis-field-engineering/redis-connect-dist/archive/main.zip && \
+mkdir -p redis-connect/demo && \
+mkdir -p redis-connect/k8s-docs && \
+unzip main.zip "redis-connect-dist-main/examples/mongodb/*" -d redis-connect && \
+cp -R redis-connect/redis-connect-dist-main/examples/mongodb/demo/* redis-connect/demo && \
+cp -R redis-connect/redis-connect-dist-main/examples/mongodb/k8s-docs/* redis-connect/k8s-docs && \
+rm -rf main.zip redis-connect/redis-connect-dist-main && \
+cd redis-connect && \
+chmod a+x demo/*.sh && \
+cd demo
+```
+
+## Setup MongoDB database (Source)
+
+https://www.mongodb.com/atlas/database
+
+## Setup Redis Enterprise cluster, databases and RedisInsight in docker (Target)
+<br>Execute [setup_re.sh](setup_re.sh)</br>
+```bash
+demo$ ./setup_re.sh
+```
+**NOTE**
+
+The above script will create a 1-node Redis Enterprise cluster in a docker container, [Create a target database with RediSearch module](https://docs.redislabs.com/latest/modules/add-module-to-database/), [Create a job management and metrics database with RedisTimeSeries module](https://docs.redislabs.com/latest/modules/add-module-to-database/), [Create a RediSearch index for emp Hash](https://redislabs.com/blog/getting-started-with-redisearch-2-0/), [Start a docker instance of grafana with Redis Data Source](https://redisgrafana.github.io/) and [Start an instance of RedisInsight](https://docs.redislabs.com/latest/ri/installing/install-docker/).
+
+## Start Redis Connect
+
+<details><summary>Review options by running Redis Connect docker container </summary>
+<p>
+
+```bash
+demo$ docker run \
+-it --rm --privileged=true \
+--name redis-connect-$(hostname) \
+-v $(pwd)/config:/opt/redislabs/redis-connect/config \
+-v $(pwd)/config/samples/credentials:/opt/redislabs/redis-connect/config/samples/credentials \
+--net host \
+redislabs/redis-connect
+```
+
+</p>
+</details>
+
+<details><summary>Expected output:</summary>
+<p>
+
+```bash
+-------------------------------
+Redis Connect startup script.
+*******************************
+Please ensure that these environment variables are correctly mapped before executing start and cli options. They can also be found in /opt/redislabs/redis-connect/bin/redisconnect.conf
+Example environment variables and volume mapping for docker based deployments
+-e REDISCONNECT_JOB_MANAGER_CONFIG_PATH=/opt/redislabs/redis-connect/config/jobmanager.properties [OPTIONAL]
+-e REDISCONNECT_LOGBACK_CONFIG=/opt/redislabs/redis-connect/config/logback.xml [OPTIONAL]
+-e REDISCONNECT_JAVA_OPTIONS=-Xms1g -Xmx2g [OPTIONAL]
+-e REDISCONNECT_EXTLIB_DIR=/opt/redislabs/redis-connect/extlib [OPTIONAL]
+-v <HOST_PATH_TO_JOB_MANAGER_PROPERTIES>:/opt/redislabs/redis-connect/config
+-v <HOST_PATH_TO_CREDENTIALS>:/opt/redislabs/redis-connect/config/samples/credentials
+-v <HOST_PATH_TO_EXTLIB>:/opt/redislabs/redis-connect/extlib [OPTIONAL]
+-p 8282:8282
+
+Usage: [-h|cli|start]
+options:
+-h: Print this help message and exit.
+-v: Print version.
+cli: init Redis Connect CLI
+start: init Redis Connect Instance (Cluster Member)
+-------------------------------
+```
+
+</p>
+</details>
+
+<details><summary><b>Start Redis Connect Instance</b></summary>
+<p>
+
+```bash
+docker run \
+-it --rm --privileged=true \
+--name redis-connect-$(hostname) \
+-v $(pwd)/config:/opt/redislabs/redis-connect/config \
+-v $(pwd)/extlib:/opt/redislabs/redis-connect/extlib \
+--net host \
+redislabs/redis-connect start
+```
+
+</p>
+</details>
+
+<details><summary>Expected output:</summary>
+<p>
+
+```bash
+
+```
+
+</p>
+</details>
+
+**Open browser to access Swagger UI -** [http://localhost:8282/swagger-ui/index.html]()
+<br>_For quick start, use '**cdc_job**' as **jobName**_
+<br><br><img src="/images/quick-start/Redis Connect Swagger Front Page.jpg" style="float: right;" width = 700px height = 425px/>
+
+**Create Job Configuration** - `/connect/api/vi/job/config/{jobName}`
+<br>_For quick start, use the sample `cdc-job.json` configuration:_ <a href="/examples/mongodb/demo/config/samples/payloads/cdc-job.json">SQL Server</a>
+<br><br><img src="/images/quick-start/Redis Connect Save Job Config.png" style="float: right;" width = 700px height = 375px/>
+<br>
+
+**Or Use `curl` to create the `cdc-job` configuration** <br>
+`demo$ curl -v -X POST "http://localhost:8282/connect/api/v1/job/config/cdc-job" -H "accept: */*" -H "Content-Type: multipart/form-data" -F "file=@config/samples/payloads/cdc-job.json;type=application/json"`
+
+-------------------------------
+
+### Initial Loader Steps
+
+**Start Job -** `/connect/api/vi/job/transition/start/{jobName}/{jobType}`
+<br>Use '**load**' as _**jobType**_
+<br><br><img src="/images/quick-start/Redis Connect Start Job.png" style="float: right;" width = 700px height = 375px/>
+
+**Or Use `curl` to start the initial load for `cdc-job`** <br>
+`demo$ curl -X POST "http://localhost:8282/connect/api/v1/job/transition/start/cdc-job/load" -H "accept: */*"`
+
+<details><summary><b>Query for the above inserted record in Redis (target)</b></summary>
+<p>
+
+```bash
+demo$ sudo docker exec -it re-node1 bash -c 'redis-cli -p 12000 ft.search idx_emp "@empno:[1 11]"'
+
+```
+
+</p>
+</details>
+
+-------------------------------
+
+### CDC Steps
+
+**Start Job -** `/connect/api/vi/job/transition/start/{jobName}/{jobType}`
+<br>Use '**stream**' as _**jobType**_
+<br><br><img src="/images/quick-start/Redis Connect Start Job.png" style="float: right;" width = 700px height = 375px/>
+
+**Or Use `curl` to start the stream for `cdc-job`** <br>
+`demo$ curl -X POST "http://localhost:8282/connect/api/v1/job/transition/start/cdc-job/stream" -H "accept: */*"`
+
+**Confirm Job Claim -** `/connect/api/vi/jobs/claim/{jobStatus}`
+<br>_For quick start, use '**all**' as **jobStatus**_
+<br><br><img src="/images/quick-start/Redis Connect Get Claims.png" style="float: right;" width = 700px height = 250px/>
+
+**Or Use `curl` to query the `cdc-job` status** <br>
+`demo$ curl -X GET "http://localhost:8282/connect/api/v1/cluster/jobs/claim/all" -H "accept: */*"`
+
+Expected output: `[{"jobId":"{connect}:job:cdc-job","jobName":"cdc-job","jobStatus":"CLAIMED","jobOwner":"30@virag-cdc","jobType":"STREAM"}]`
+
+<b>INSERT/UPDATE a document in MongoDB (source)</b>
+<p>
+
+<details><summary><b>Query for the above inserted record in Redis (target)</b></summary>
+<p>
+
+```bash
+demo$ sudo docker exec -it re-node1 bash -c 'redis-cli -p 12000 ft.search idx_emp "@empno:[1002 1002]"'
+
+```
+
+</p>
+</details>

examples/mongodb/demo/config/jobmanager.properties

@@ -0,0 +1,38 @@
+##################################################################################################
+#####     Properties to start a Redis Connect cluster. Default values are commented out.     #####
+##################################################################################################
+
+#####     Cluster properties
+#cluster.name=default
+#cluster.leader.heartbeat.lease.renewal.interval=5000
+#cluster.election.attempt.interval=5000
+#cluster.timeseries.metrics.enabled=false
+
+#####     Job Manager Services properties
+#job.manager.services.enabled=true
+#job.manager.services.threadpool.size=2
+#job.reap.attempt.interval=7000
+#job.claim.service.enabled=true
+#job.claim.attempt.interval=5000
+#job.claim.batch.size.per.attempt=4
+#job.claim.max.capacity=4
+#job.claim.heartbeat.lease.renewal.interval=10000
+
+#####     REST properties
+#rest.api.enabled=true
+#rest.api.port=8282
+
+#####     Job Manager Database properties
+redis.connection.url=redis://127.0.0.1:14001
+#redis.connection.sslEnabled=false
+#redis.truststore=<path_to_JKS>
+#redis.keystore=<path_to_PKCS>
+#redis.connection.insecure=false
+#redis.connection.timeout.duration=5
+#redis.connection.auto.reconnect=true
+#redis.connection.suspend.reconnect.on.protocol.failure=true
+
+#####     Credentials properties
+credentials.file.path=/opt/redislabs/redis-connect/config/samples/credentials
+#credentials.rotation.event.listener.enabled=false
+#credentials.rotation.event.listener.interval=60000

examples/mongodb/demo/config/logback-cli.xml

@@ -0,0 +1,113 @@
+<configuration debug="false" scan="true" scanPeriod="15 seconds">
+
+    <property name="LOG_REDIS_CONNECT_PATH" value="logs/redis-connect"/>
+    <property name="LOG_REDIS_CONNECT_MANAGER_PATH" value="logs/redis-connect-manager.log"/>
+    <property name="LOG_REDIS_CONNECT_HEARTBEAT_PATH" value="logs/redis-connect-heartbeat.log"/>
+
+    <appender name="REDIS_CONNECT_HEARTBEAT" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_REDIS_CONNECT_HEARTBEAT_PATH}</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>logs/archived/redis-connect-heartbeat.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each archived file, size max 10MB -->
+            <maxFileSize>10MB</maxFileSize>
+            <!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
+            <totalSizeCap>20GB</totalSizeCap>
+            <!-- 60 days to keep -->
+            <maxHistory>60</maxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d %p %c{1.} [%t] %m%n</pattern>
+        </encoder>
+    </appender>
+    <appender name="REDIS_CONNECT_MANAGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_REDIS_CONNECT_MANAGER_PATH}</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>logs/archived/redis-connect-manager.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each archived file, size max 10MB -->
+            <maxFileSize>10MB</maxFileSize>
+            <!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
+            <totalSizeCap>20GB</totalSizeCap>
+            <!-- 60 days to keep -->
+            <maxHistory>60</maxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d %p %c{1.} [%t] %m%n</pattern>
+        </encoder>
+    </appender>
+    <appender name="REDIS_CONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_REDIS_CONNECT_PATH}-${PID}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>logs/archived/redis-connect-${PID}.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <!-- each archived file, size max 10MB -->
+            <maxFileSize>10MB</maxFileSize>
+            <!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
+            <totalSizeCap>20GB</totalSizeCap>
+            <!-- 60 days to keep -->
+            <maxHistory>60</maxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>%d %p %c{1.} [%t] %m%n</pattern>
+        </encoder>
+    </appender>
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="redis-connect-heartbeat" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT_HEARTBEAT"/>
+        <appender-ref ref="CONSOLE" />
+    </logger>
+    <logger name="redis-connect-manager" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT_MANAGER"/>
+        <appender-ref ref="CONSOLE" />
+    </logger>
+    <logger name="redis-connect" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE" />
+    </logger>
+    <logger name="io.netty" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE" />
+    </logger>
+    <logger name="io.lettuce" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE" />
+    </logger>
+    <logger name="com.zaxxer" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="io.debezium" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.apache.kafka" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.springframework" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="springfox" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org" level="OFF" additivity="false">
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="Validator" level="OFF" additivity="false"/>
+    <logger name="_org.springframework.web.servlet.HandlerMapping" level="OFF" additivity="false"/>
+    <logger name="com.redis.connect.cluster.RedisConnectSpringBootApplication" level="OFF" additivity="false"/>
+    <logger name="reactor.util" level="OFF" additivity="false"/>
+
+    <root>
+        <appender-ref ref="REDIS_CONNECT"/>
+        <appender-ref ref="REDIS_CONNECT_MANAGER"/>
+        <appender-ref ref="REDIS_CONNECT_HEARTBEAT"/>
+    </root>
+
+</configuration>