Add new templates with zstd support

Signed-off-by: Mohit Godwani <[email protected]>
opensearch-project · Aug 19, 2024 · eefde3c · eefde3c
1 parent cd264ab
commit eefde3c
Show file tree

Hide file tree

Showing 14 changed files with 5,272 additions and 148 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
diff --git a/build.gradle b/build.gradle
@@ -67,6 +67,10 @@ repositories {
     maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
 }
 
+configurations {
+    zipArchive
+}
+
 allprojects {
     group 'org.opensearch'
     version = opensearch_version.tokenize('-')[0] + '.0'
@@ -92,6 +96,7 @@ opensearchplugin {
 }
 
 dependencies {
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-custom-codecs', version: "3.0.0.0-SNAPSHOT"
 }
 
 allprojects {
@@ -148,6 +153,7 @@ publishing {
                 password "$System.env.SONATYPE_PASSWORD"
             }
         }
+        mavenLocal()
     }
 }
 
@@ -202,14 +208,27 @@ integTest {
     dependsOn "bundlePlugin"
     systemProperty 'tests.security.manager', 'true'
 
+
     systemProperty "https", System.getProperty("https")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 }
 
 testClusters.integTest {
-    testDistribution = "ARCHIVE"
+    testDistribution = "INTEG_TEST"
     plugin(project.tasks.bundlePlugin.archiveFile)
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() {
+                return configurations.zipArchive.asFileTree.matching {
+                    include '**/opensearch-custom-codecs*'
+                }.singleFile
+            }
+        }
+    }))
+    systemProperty 'opensearch.experimental.feature.application_templates.enabled', 'true'
+    setting 'cluster.application_templates.enabled', 'true'
 }
 
 tasks.withType(PublishToMavenRepository) {

diff --git a/.../org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java b/.../org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextTemplateIT.java
@@ -20,10 +20,14 @@
 import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
 import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.opensearch.client.Request;
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 import javax.net.ssl.SSLEngine;
@@ -32,15 +36,50 @@
 import java.security.KeyManagementException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE;
 import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL;
 
 public class CreateIndexTemplateWithContextTemplateIT extends OpenSearchRestTestCase {
 
+    @SuppressWarnings("unchecked")
     public void testCreateIndexWithContextBasedTemplate() throws IOException {
-        // TODO: Add E2E test with rest layer here.
+
+        final String indexTemplate = "my-metrics-template";
+        final String index = "my-metrics-1";
+
+        Request request = new Request("PUT", "/_index_template/" + indexTemplate);
+        String content = "{\n"
+            + "    \"index_patterns\": [\n"
+            + "        \"my-metrics-*\"\n"
+            + "    ],\n"
+            + "    \"context\": {\n"
+            + "        \"name\": \"metrics\"\n"
+            + "    }\n"
+            + "}";
+
+        request.setJsonEntity(content);
+
+        // Create index template
+        client().performRequest(request);
+
+        // creating index
+        createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build());
+
+        Map<String, Object> currentIndexSettings = (Map<String, Object>) ((Map<String, Object>) getIndexSettings(index).get(index)).get(
+            "settings"
+        );
+        assertEquals(currentIndexSettings.get(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()), "60s");
+        assertEquals(currentIndexSettings.get(IndexSettings.INDEX_MERGE_POLICY.getKey()), "log_byte_size");
+        assertEquals(currentIndexSettings.get(EngineConfig.INDEX_CODEC_SETTING.getKey()), "zstd_no_dict");
+
+        try {
+            ensureGreen(index);
+        } finally {
+            deleteIndex(index);
+        }
     }
 
     @Override

diff --git a/...est/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java b/...est/java/org/opensearch/system/applicationtemplates/CreateIndexTemplateWithContextIT.java
diff --git a/src/main/resources/org/opensearch/system/applicationtemplates/templates.json b/src/main/resources/org/opensearch/system/applicationtemplates/templates.json
@@ -1,15 +1,40 @@
 {
   "repository_schema_version": 1,
   "templates": [
-      {
-        "name": "logs",
-        "version": 1,
-        "type": "@abc_template"
-      },
-      {
-        "name": "metrics",
-        "version": 1,
-        "type": "@abc_template"
-      }
+    {
+      "name": "logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "metrics",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "apache-web-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "aws-cloudtrail-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "aws-elb-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "aws-s3-logs",
+      "version": 1,
+      "type": "@abc_template"
+    },
+    {
+      "name": "nginx-logs",
+      "version": 1,
+      "type": "@abc_template"
+    }
   ]
 }