Skip to content

Commit

Permalink
Pulling in secuirty as a zipArchive dependency, installed and configu…
Browse files Browse the repository at this point in the history 
…red only for security y enabled clusters

Signed-off-by: Joshua Palis <[email protected]>
  • Loading branch information
@joshpalis
joshpalis committed Jan 12, 2024
1 parent 6b2dfb3 commit 714ee34
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 209 deletions.
83 changes: 24 additions & 59 deletions .github/workflows/test_security.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,74 +8,39 @@ on:
- "*"

jobs:
Build-ad:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

integ-test-with-security-linux:
strategy:
matrix:
java: [11,17,21]
fail-fast: false
java: [11, 17, 21]

name: Security test workflow for Flow Framework
name: Run Secuirty Integration Tests on Linux
runs-on: ubuntu-latest
needs: Get-CI-Image-Tag
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

steps:
- name: Checkout Flow Framework
uses: actions/checkout@v4
with:
submodules: true

- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

- name: Checkout Flow Framework
uses: actions/checkout@v4

- name: Build Flow Framework
run: |
./gradlew assemble
# example of variables:
# plugin = opensearch-flow-framework-2.10.0.0-SNAPSHOT.zip
# version = 2.10.0, plugin_version = 2.10.0.0, qualifier = SNAPSHOT
- name: Pull and Run Docker
run: |
plugin=`basename $(ls build/distributions/*.zip)`
version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
qualifier=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
if $qualifier!=SNAPSHOT
then
docker_version=$version-$qualifier
else
docker_version=$version
fi
echo plugin version plugin_version qualifier docker_version
echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
cd ..
if docker pull opensearchstaging/opensearch:$docker_version
then
echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-flow-framework ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-flow-framework; fi" >> Dockerfile
echo "ADD flow-framework/build/distributions/$plugin /tmp/" >> Dockerfile
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
docker build -t opensearch-flow-framework:test .
echo "imagePresent=true" >> $GITHUB_ENV
else
echo "imagePresent=false" >> $GITHUB_ENV
fi
- name: Run Docker Image
if: env.imagePresent == 'true'
run: |
cd ..
docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-flow-framework:test
sleep 90
- name: Run Flow Framework Test
if: env.imagePresent == 'true'
- name: Run build
# switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
run: |
security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
if [ $security -gt 0 ]
then
echo "Security plugin is available"
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
else
echo "Security plugin is NOT available, skipping integration tests"
fi
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dhttps=true -Duser=admin -Dpassword=admin"
197 changes: 151 additions & 46 deletions build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,48 @@
import java.nio.file.Files
import org.opensearch.gradle.testclusters.OpenSearchCluster
import org.opensearch.gradle.test.RestIntegTestTask
import java.util.concurrent.Callable
import java.nio.file.Paths

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
version_tokens = opensearch_version.tokenize('-')
opensearch_build = version_tokens[0] + '.0'
plugin_no_snapshot = opensearch_build
if (buildVersionQualifier) {
opensearch_build += "-${buildVersionQualifier}"
plugin_no_snapshot += "-${buildVersionQualifier}"
}
if (isSnapshot) {
opensearch_build += "-SNAPSHOT"
}
opensearch_group = "org.opensearch"
opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
System.setProperty('tests.security.manager', 'false')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
}

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
maven { url 'https://jitpack.io' }
}

dependencies {
classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.3"
classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
}
}

plugins {
id "de.undercouch.download" version "5.3.0"
}

apply plugin: 'java'
apply plugin: 'idea'
Expand Down Expand Up @@ -39,42 +81,6 @@ thirdPartyAudit.enabled = false
// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
version_tokens = opensearch_version.tokenize('-')
opensearch_build = version_tokens[0] + '.0'
plugin_no_snapshot = opensearch_build
if (buildVersionQualifier) {
opensearch_build += "-${buildVersionQualifier}"
plugin_no_snapshot += "-${buildVersionQualifier}"
}
if (isSnapshot) {
opensearch_build += "-SNAPSHOT"
}
opensearch_group = "org.opensearch"
opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
System.setProperty('tests.security.manager', 'false')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
}

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
maven { url 'https://jitpack.io' }
}

dependencies {
classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
classpath "com.diffplug.spotless:spotless-plugin-gradle:6.23.3"
classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
}
}

allprojects {
// Default to the apache license
project.ext.licenseName = 'The Apache Software License, Version 2.0'
Expand Down Expand Up @@ -156,6 +162,7 @@ dependencies {

// ZipArchive dependencies used for integration tests
zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"

configurations.all {
resolutionStrategy {
Expand All @@ -171,6 +178,96 @@ def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absolu
opensearch_tmp_dir.mkdirs()
def _numNodes = findProperty('numNodes') as Integer ?: 1

ext{

configureSecurityPlugin = { OpenSearchCluster cluster ->

// Retrieve Security Plugin Zip from zipArchive
configurations.zipArchive.asFileTree.each {
if(it.name.contains("opensearch-security")) {
cluster.plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
})
)
}
}

cluster.getNodes().forEach { node ->
var creds = node.getCredentials()
if (creds.isEmpty()) {
creds.add(Map.of('username', 'admin', 'password', 'admin'))
} else {
creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
}
}

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
cluster.extraConfigFile(file, local)
}

// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
cluster.setting("plugins.security.ssl.http.enabled", "true")
cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
cluster.setting("plugins.security.allow_default_init_securityindex", "true")
cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")

cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
cluster.setting('plugins.security.system_indices.enabled', "true")
cluster.setting('plugins.security.system_indices.indices', '[' +
'".plugins-ml-config", ' +
'".plugins-ml-connector", ' +
'".plugins-ml-model-group", ' +
'".plugins-ml-model", ".plugins-ml-task", ' +
'".plugins-ml-conversation-meta", ' +
'".plugins-ml-conversation-interactions", ' +
'".opendistro-alerting-config", ' +
'".opendistro-alerting-alert*", ' +
'".opendistro-anomaly-results*", ' +
'".opendistro-anomaly-detector*", ' +
'".opendistro-anomaly-checkpoints", ' +
'".opendistro-anomaly-detection-state", ' +
'".opendistro-reports-*", ' +
'".opensearch-notifications-*", ' +
'".opensearch-notebooks", ' +
'".opensearch-observability", ' +
'".ql-datasources", ' +
'".opendistro-asynchronous-search-response*", ' +
'".replication-metadata-store", ' +
'".opensearch-knn-models", ' +
'".geospatial-ip2geo-data*", ' +
'".plugins-flow-framework-config", ' +
'".plugins-flow-framework-templates", ' +
'".plugins-flow-framework-state"' +
']'
)
cluster.setSecure(true)
}
}

test {
include '**/*Tests.class'
}
Expand Down Expand Up @@ -249,19 +346,27 @@ integTest {
testClusters.integTest {
testDistribution = "ARCHIVE"

// Installs all registered zipArchive dependencies on integTest cluster nodes
// Optionally install security
if (System.getProperty("https") != null && System.getProperty("https") == "true") {
configureSecurityPlugin(testClusters.integTest)
}

// Installs all registered zipArchive dependencies on integTest cluster nodes except security
configurations.zipArchive.asFileTree.each {
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
if(!it.name.contains("opensearch-security")) {
plugin(provider(new Callable<RegularFile>(){
@Override
File getAsFile() {
return it
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
}
}
}))
})
)
}
}

// Install Flow Framwork Plugin on integTest cluster nodes
Expand Down
Loading

0 comments on commit 714ee34

Please sign in to comment.