Skip to content

Commit

Permalink
Fix various issues (see description) (#89)
Browse files Browse the repository at this point in the history
* Expand ConfigProvider to load from a namespace

* Fix snakeyaml issue

* sqljob update
  • Loading branch information
jogrogan authored Jan 21, 2025
1 parent 0e19953 commit ed98677
Show file tree
Hide file tree
Showing 58 changed files with 133 additions and 164 deletions.
1 change: 0 additions & 1 deletion deploy/samples/demodb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,3 @@ spec:
- Modify
connector: |
connector = blackhole
87 changes: 0 additions & 87 deletions deploy/sqljobs.crd.yaml

This file was deleted.

5 changes: 3 additions & 2 deletions generate-models.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ docker run \
ghcr.io/kubernetes-client/java/crd-model-gen:v1.0.6 \
/generate.sh -o "$(pwd)/hoptimator-k8s" -n "" -p "com.linkedin.hoptimator.k8s" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/databases.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/engines.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/jobtemplates.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/pipelines.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/subscriptions.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/tabletemplates.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/views.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/subscriptions.crd.yaml" \
-u "$(pwd)/hoptimator-k8s/src/main/resources/engines.crd.yaml" \
&& echo "done."
7 changes: 4 additions & 3 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@ avro = "org.apache.avro:avro:1.10.2"
calcite-avatica = "org.apache.calcite.avatica:avatica:1.23.0"
calcite-core = "org.apache.calcite:calcite-core:1.34.0"
calcite-server = "org.apache.calcite:calcite-server:1.34.0"
commons-cli = "commons-cli:commons-cli:1.4"
flink-clients = "org.apache.flink:flink-clients:1.18.1"
flink-connector-base = "org.apache.flink:flink-connector-base:1.18.1"
flink-connector-kafka = "org.apache.flink:flink-sql-connector-kafka:3.2.0-1.18"
flink-connector-mysql-cdc = "com.ververica:flink-sql-connector-mysql-cdc:2.3.0"
flink-core = "org.apache.flink:flink-core:1.18.1"
flink-csv = "org.apache.flink:flink-csv:1.18.1"
flink-jdbc = "org.apache.flink:flink-sql-jdbc-driver-bundle:1.18.1"
Expand All @@ -15,8 +18,6 @@ flink-table-api-java-bridge = "org.apache.flink:flink-table-api-java-bridge:1.18
flink-table-common = "org.apache.flink:flink-table-common:1.18.1"
flink-table-planner = "org.apache.flink:flink-table-planner_2.12:1.18.1"
flink-table-runtime = "org.apache.flink:flink-table-runtime:1.18.1"
flink-connector-kafka = "org.apache.flink:flink-sql-connector-kafka:3.2.0-1.18"
flink-connector-mysql-cdc = "com.ververica:flink-sql-connector-mysql-cdc:2.3.0"
gson = "com.google.code.gson:gson:2.9.0"
jackson = "com.fasterxml.jackson.core:jackson-core:2.14.1"
jackson-dataformat-yaml = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.1"
Expand All @@ -27,8 +28,8 @@ kubernetes-client = "io.kubernetes:client-java:16.0.2"
kubernetes-extended-client = "io.kubernetes:client-java-extended:16.0.2"
slf4j-simple = "org.slf4j:slf4j-simple:1.7.30"
slf4j-api = "org.slf4j:slf4j-api:1.7.30"
snakeyaml = "org.yaml:snakeyaml:1.33"
sqlline = "sqlline:sqlline:1.12.0"
commons-cli = "commons-cli:commons-cli:1.4"
quidem = "net.hydromatic:quidem:0.11"
venice = "com.linkedin.venice:venice-common:0.4.376"
venice-client = "com.linkedin.venice:venice-thin-client:0.4.376"
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@

public interface ConfigProvider {

Properties loadConfig() throws Exception;
Properties loadConfig(String namespace) throws Exception;
}
2 changes: 1 addition & 1 deletion hoptimator-flink-adapter/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ plugins {

dependencies {
implementation project(':hoptimator-catalog')
implementation project(':hoptimator-models')
implementation project(':hoptimator-k8s')
implementation project(':hoptimator-operator')
implementation libs.kubernetesClient
implementation libs.kubernetesExtendedClient
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.linkedin.hoptimator.operator.flink;

import com.linkedin.hoptimator.models.V1alpha1SqlJob;
import com.linkedin.hoptimator.models.V1alpha1SqlJobList;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJob;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobList;
import com.linkedin.hoptimator.operator.ControllerProvider;
import com.linkedin.hoptimator.operator.Operator;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

import com.linkedin.hoptimator.catalog.Resource;
import com.linkedin.hoptimator.catalog.flink.FlinkStreamingSqlJob;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJob;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobSpec.DialectEnum;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobSpec.ExecutionModeEnum;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobStatus;
import com.linkedin.hoptimator.operator.Operator;
import com.linkedin.hoptimator.models.V1alpha1SqlJob;
import com.linkedin.hoptimator.models.V1alpha1SqlJobSpec.DialectEnum;
import com.linkedin.hoptimator.models.V1alpha1SqlJobSpec.ExecutionModeEnum;
import com.linkedin.hoptimator.models.V1alpha1SqlJobStatus;

import io.kubernetes.client.extended.controller.reconciler.Reconciler;
import io.kubernetes.client.extended.controller.reconciler.Request;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

public class SystemPropertiesConfigProvider implements ConfigProvider {

public Properties loadConfig() {
public Properties loadConfig(String namespace) {
return System.getProperties();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,19 @@ public class K8sConfigProvider implements ConfigProvider {

public static final String HOPTIMATOR_CONFIG_MAP = "hoptimator-configmap";

public Properties loadConfig() throws SQLException {
Map<String, String> topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP);
public Properties loadConfig(String namespace) throws SQLException {
Map<String, String> topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP, namespace);
Properties p = new Properties();
p.putAll(topLevelConfigs);
return p;
}

// Load top-level config map properties
private Map<String, String> loadTopLevelConfig(String configMapName) throws SQLException {
private Map<String, String> loadTopLevelConfig(String configMapName, String namespace) throws SQLException {
K8sApi<V1ConfigMap, V1ConfigMapList> configMapApi = new K8sApi<>(K8sContext.currentContext(), K8sApiEndpoints.CONFIG_MAPS);
return configMapApi.get(configMapName).getData();
if (namespace == null || namespace.isEmpty()) {
return configMapApi.get(configMapName).getData();
}
return configMapApi.get(namespace, configMapName).getData();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* Database metadata.
*/
@ApiModel(description = "Database metadata.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1Database implements io.kubernetes.client.common.KubernetesObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
* DatabaseList is a list of Database
*/
@ApiModel(description = "DatabaseList is a list of Database")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1DatabaseList implements io.kubernetes.client.common.KubernetesListObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* Database spec.
*/
@ApiModel(description = "Database spec.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1DatabaseSpec {
/**
* SQL dialect the driver expects.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* Engine metadata.
*/
@ApiModel(description = "Engine metadata.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1Engine implements io.kubernetes.client.common.KubernetesObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
* EngineList is a list of Engine
*/
@ApiModel(description = "EngineList is a list of Engine")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1EngineList implements io.kubernetes.client.common.KubernetesListObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* Engine spec.
*/
@ApiModel(description = "Engine spec.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1EngineSpec {
public static final String SERIALIZED_NAME_DATABASES = "databases";
@SerializedName(SERIALIZED_NAME_DATABASES)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* Template to apply to matching jobs.
*/
@ApiModel(description = "Template to apply to matching jobs.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1JobTemplate implements io.kubernetes.client.common.KubernetesObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
* JobTemplateList is a list of JobTemplate
*/
@ApiModel(description = "JobTemplateList is a list of JobTemplate")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1JobTemplateList implements io.kubernetes.client.common.KubernetesListObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* TableTemplate spec.
*/
@ApiModel(description = "TableTemplate spec.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1JobTemplateSpec {
public static final String SERIALIZED_NAME_DATABASES = "databases";
@SerializedName(SERIALIZED_NAME_DATABASES)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
* A set of objects that work together to deliver data.
*/
@ApiModel(description = "A set of objects that work together to deliver data.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1Pipeline implements io.kubernetes.client.common.KubernetesObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
* PipelineList is a list of Pipeline
*/
@ApiModel(description = "PipelineList is a list of Pipeline")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1PipelineList implements io.kubernetes.client.common.KubernetesListObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* Pipeline spec.
*/
@ApiModel(description = "Pipeline spec.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1PipelineSpec {
public static final String SERIALIZED_NAME_SQL = "sql";
@SerializedName(SERIALIZED_NAME_SQL)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* Pipeline status.
*/
@ApiModel(description = "Pipeline status.")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-14T23:39:16.570Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1PipelineStatus {
public static final String SERIALIZED_NAME_FAILED = "failed";
@SerializedName(SERIALIZED_NAME_FAILED)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
*/


package com.linkedin.hoptimator.models;
package com.linkedin.hoptimator.k8s.models;

import java.util.Objects;
import java.util.Arrays;
Expand All @@ -20,8 +20,8 @@
import com.google.gson.annotations.SerializedName;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import com.linkedin.hoptimator.models.V1alpha1SqlJobSpec;
import com.linkedin.hoptimator.models.V1alpha1SqlJobStatus;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobSpec;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJobStatus;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
Expand All @@ -31,7 +31,7 @@
* Hoptimator generic SQL job
*/
@ApiModel(description = "Hoptimator generic SQL job")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:55:33.927Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1SqlJob implements io.kubernetes.client.common.KubernetesObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
*/


package com.linkedin.hoptimator.models;
package com.linkedin.hoptimator.k8s.models;

import java.util.Objects;
import java.util.Arrays;
Expand All @@ -20,7 +20,7 @@
import com.google.gson.annotations.SerializedName;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import com.linkedin.hoptimator.models.V1alpha1SqlJob;
import com.linkedin.hoptimator.k8s.models.V1alpha1SqlJob;
import io.kubernetes.client.openapi.models.V1ListMeta;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
Expand All @@ -32,7 +32,7 @@
* SqlJobList is a list of SqlJob
*/
@ApiModel(description = "SqlJobList is a list of SqlJob")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2024-12-09T16:55:33.927Z[Etc/UTC]")
@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2025-01-20T21:16:25.561Z[Etc/UTC]")
public class V1alpha1SqlJobList implements io.kubernetes.client.common.KubernetesListObject {
public static final String SERIALIZED_NAME_API_VERSION = "apiVersion";
@SerializedName(SERIALIZED_NAME_API_VERSION)
Expand Down
Loading

0 comments on commit ed98677

Please sign in to comment.