From 657c106935055bda19ee040b7dc81467821d9e83 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Sun, 23 Dec 2018 21:56:18 +0100 Subject: [PATCH 1/4] Reduced cyclomatic complexity (quit fast) --- bootstrap.sh | 305 +++++++++++++++++++++++++-------------------------- 1 file changed, 150 insertions(+), 155 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 69b845e..2a3b13a 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -4,161 +4,156 @@ set -e echo '===> DSE Configuration' # See if we've already completed bootstrapping -if [ ! -f killrvideo_bootstrapped ]; then - - # Default addresses to use for DSE cluster if starting in Docker - dse_ip='dse' - dse_external_ip=$KILLRVIDEO_DOCKER_IP - dse_enable_ssl='false' - - # Create cql_options variable to consolidate multiple options into one - # variable for easier reading - cql_options='' - # Use space variable to concatenate options - space=' ' - - # If an external cluster address is provided, use that - if [ ! -z "$KILLRVIDEO_DSE_EXTERNAL_IP" ]; then - dse_ip=$KILLRVIDEO_DSE_EXTERNAL_IP - dse_external_ip=$KILLRVIDEO_DSE_EXTERNAL_IP - fi - echo "=> Setting up KillrVideo via DSE node at: $dse_ip" - - # If a request timeout is available use that. This is useful - # in cases where a longer timeout is needed for cqlsh operations - if [ ! -z "$KILLRVIDEO_DSE_REQUEST_TIMEOUT" ]; then - dse_request_timeout="--request-timeout=$KILLRVIDEO_DSE_REQUEST_TIMEOUT --connect-timeout=$KILLRVIDEO_DSE_REQUEST_TIMEOUT" - cql_options="$dse_request_timeout" - - echo "=> Request timeout set at: $dse_request_timeout" - fi - - # If SSL is enabled, then provide SSL info - if [ "$KILLRVIDEO_ENABLE_SSL" = 'true' ]; then - dse_enable_ssl='true' - - # The reference to this file is provided via a volume enabled - # on the dse-config container within docker-compose.yaml - # in the killrvideo-docker-common repo - dse_ssl_certfile='/opt/killrvideo-data/cassandra.cert' - dse_ssl='--ssl' - cql_options="$cql_options$space$dse_ssl" - - # These 2 environment variables are needed for cqlsh to - # properly handle SSL - export SSL_CERTFILE=$dse_ssl_certfile - export SSL_VALIDATE=true - - echo "=> SSL encryption is ENABLED with CERT FILE: $dse_ssl_certfile" - fi - - # Wait for port 9042 (CQL) to be ready for up to 240 seconds - echo '=> Waiting for DSE to become available' - /wait-for-it.sh -t 300 $dse_ip:9042 - echo '=> DSE is available' - echo "=> If any exist, cql_options are: $cql_options" - - # Default privileges - admin_user='cassandra' - admin_password='cassandra' - dse_user='cassandra' - dse_password='cassandra' - - # If requested, create a new superuser to replace the default superuser - if [ "$KILLRVIDEO_CREATE_ADMIN_USER" = 'true' ]; then - echo "=> Creating new superuser $KILLRVIDEO_ADMIN_USERNAME" - cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "CREATE ROLE $KILLRVIDEO_ADMIN_USERNAME with SUPERUSER = true and LOGIN = true and PASSWORD = '$KILLRVIDEO_ADMIN_PASSWORD'" - # Login as new superuser to delete default superuser (cassandra) - cqlsh $dse_ip 9042 -u $KILLRVIDEO_ADMIN_USERNAME -p $KILLRVIDEO_ADMIN_PASSWORD $cql_options -e "DROP ROLE $admin_user" - fi - - # Use new admin credentials for future actions - if [ ! -z "$KILLRVIDEO_ADMIN_USERNAME" ]; then - admin_user=$KILLRVIDEO_ADMIN_USERNAME - admin_password=$KILLRVIDEO_ADMIN_PASSWORD - fi - - # If requested, create a new standard user - if [ "$KILLRVIDEO_CREATE_DSE_USER" = 'true' ]; then - # Create user and grant permission to create keyspaces (generator and web will need) - echo "=> Creating user $KILLRVIDEO_DSE_USERNAME and granting keyspace creation permissions" - cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "CREATE ROLE $KILLRVIDEO_DSE_USERNAME with LOGIN = true and PASSWORD = '$KILLRVIDEO_DSE_PASSWORD'" - echo '=> Granting keyspace creation permissions' - cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "GRANT CREATE on ALL KEYSPACES to $KILLRVIDEO_DSE_USERNAME" - cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "GRANT ALL PERMISSIONS on ALL SEARCH INDICES to $KILLRVIDEO_DSE_USERNAME" - fi - - # Use the provided username/password for subsequent non-admin operations - if [ ! -z "$KILLRVIDEO_DSE_USERNAME" ]; then - dse_user=$KILLRVIDEO_DSE_USERNAME - dse_password=$KILLRVIDEO_DSE_PASSWORD - fi - - # Create the keyspace if necessary - echo '=> Ensuring keyspace is created' - keyspace_file='/opt/killrvideo-data/keyspace.cql' - if [ ! -z "$KILLRVIDEO_CASSANDRA_REPLICATION" ]; then - # TODO: check for valid replication format? https://stackoverflow.com/questions/21112707/check-if-a-string-matches-a-regex-in-bash-script - sed -i "s/{.*}/$KILLRVIDEO_CASSANDRA_REPLICATION/;" $keyspace_file - fi - cqlsh $dse_ip 9042 -f $keyspace_file -u $dse_user -p $dse_password $cql_options - - # TODO: Complete nodesync section once documentation is available - # Once we create the keyspace enable nodesync - # Commenting this out for now until we can get the correct - # documentation needed for using nodesync over SSL - #echo '=> Enabling NodeSync for KillrVideo keyspace' - #/opt/dse/resources/cassandra/bin/nodesync -cu $dse_user -cp $dse_password -h $dse_ip --cql-ssl enable -v -k killrvideo "*" - - # Create the schema if necessary - echo '=> Ensuring schema is created' - cqlsh $dse_ip 9042 -f /opt/killrvideo-data/schema.cql -k killrvideo -u $dse_user -p $dse_password $cql_options - - # Create DSE Search core if necessary - echo '=> Ensuring DSE Search is configured' - # TODO: temp workaround - if search index already exists, ALTER statements will cause non-zero exit - set +e - cqlsh $dse_ip 9042 -f /opt/killrvideo-data/videos_search.cql -k killrvideo -u $dse_user -p $dse_password $cql_options - # TODO: remove workaround - set -e - - # Wait for port 8182 (Gremlin) to be ready for up to 120 seconds - echo '=> Waiting for DSE Graph to become available' - /wait-for-it.sh -t 120 $dse_ip:8182 - echo '=> DSE Graph is available' - - # Update the gremlin-console remote.yaml file to set the remote hosts, username, and password - # This is required because the "dse gremlin-console" command does not accept username/password via command line - echo '=> Setting up remote.yaml for gremlin-console' - sed -i "s/.*hosts:.*/hosts: [$dse_ip]/;s/.*username:.*/username: $dse_user/;s/.*password:.*/password: $dse_password/;s|enableSsl:.*|enableSsl: $dse_enable_ssl, trustCertChainFile: $dse_ssl_certfile,|;" /opt/dse/resources/graph/gremlin-console/conf/remote.yaml - - # Create the graph if necessary - echo '=> Ensuring graph is created' - graph_file='/opt/killrvideo-data/killrvideo_video_recommendations_schema.groovy' - if [ ! -z "$KILLRVIDEO_GRAPH_REPLICATION" ]; then - sed -i "s/{.*}/$KILLRVIDEO_GRAPH_REPLICATION/;" $graph_file - fi - dse gremlin-console -e $graph_file - - # Register services in ETCD once all the schema are configured, using an IP that will be accessible - # internally and externally to the Docker environment - hostname="$(hostname)" - - echo '=> Registering DSE DB (Cassandra) cluster in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/cassandra/$hostname" -XPUT -d value="$dse_external_ip:9042" - - echo '=> Registering DSE Search in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/dse-search/$hostname" -XPUT -d value="$dse_external_ip:8983" - - echo '=> Registering DSE Graph in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/gremlin/$hostname" -XPUT -d value="$dse_external_ip:8182" - - # Don't bootstrap next time we start - echo '=> Configuration of DSE users and schema complete' - touch killrvideo_bootstrapped - -else - +if [ -f killrvideo_bootstrapped ]; then echo '=> Configuration already completed, exiting' +fi + +# Default addresses to use for DSE cluster if starting in Docker +dse_ip='dse' +dse_external_ip=$KILLRVIDEO_DOCKER_IP +dse_enable_ssl='false' + +# Create cql_options variable to consolidate multiple options into one +# variable for easier reading +cql_options='' +# Use space variable to concatenate options +space=' ' + +# If an external cluster address is provided, use that +if [ ! -z "$KILLRVIDEO_DSE_EXTERNAL_IP" ]; then + dse_ip=$KILLRVIDEO_DSE_EXTERNAL_IP + dse_external_ip=$KILLRVIDEO_DSE_EXTERNAL_IP +fi +echo "=> Setting up KillrVideo via DSE node at: $dse_ip" + +# If a request timeout is available use that. This is useful +# in cases where a longer timeout is needed for cqlsh operations +if [ ! -z "$KILLRVIDEO_DSE_REQUEST_TIMEOUT" ]; then + dse_request_timeout="--request-timeout=$KILLRVIDEO_DSE_REQUEST_TIMEOUT --connect-timeout=$KILLRVIDEO_DSE_REQUEST_TIMEOUT" + cql_options="$dse_request_timeout" + + echo "=> Request timeout set at: $dse_request_timeout" +fi + +# If SSL is enabled, then provide SSL info +if [ "$KILLRVIDEO_ENABLE_SSL" = 'true' ]; then + dse_enable_ssl='true' + + # The reference to this file is provided via a volume enabled + # on the dse-config container within docker-compose.yaml + # in the killrvideo-docker-common repo + dse_ssl_certfile='/opt/killrvideo-data/cassandra.cert' + dse_ssl='--ssl' + cql_options="$cql_options$space$dse_ssl" + + # These 2 environment variables are needed for cqlsh to + # properly handle SSL + export SSL_CERTFILE=$dse_ssl_certfile + export SSL_VALIDATE=true + echo "=> SSL encryption is ENABLED with CERT FILE: $dse_ssl_certfile" +fi + +# Wait for port 9042 (CQL) to be ready for up to 240 seconds +echo '=> Waiting for DSE to become available' +/wait-for-it.sh -t 300 $dse_ip:9042 +echo '=> DSE is available' +echo "=> If any exist, cql_options are: $cql_options" + +# Default privileges +admin_user='cassandra' +admin_password='cassandra' +dse_user='cassandra' +dse_password='cassandra' + +# If requested, create a new superuser to replace the default superuser +if [ "$KILLRVIDEO_CREATE_ADMIN_USER" = 'true' ]; then + echo "=> Creating new superuser $KILLRVIDEO_ADMIN_USERNAME" + cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "CREATE ROLE $KILLRVIDEO_ADMIN_USERNAME with SUPERUSER = true and LOGIN = true and PASSWORD = '$KILLRVIDEO_ADMIN_PASSWORD'" + # Login as new superuser to delete default superuser (cassandra) + cqlsh $dse_ip 9042 -u $KILLRVIDEO_ADMIN_USERNAME -p $KILLRVIDEO_ADMIN_PASSWORD $cql_options -e "DROP ROLE $admin_user" +fi + +# Use new admin credentials for future actions +if [ ! -z "$KILLRVIDEO_ADMIN_USERNAME" ]; then + admin_user=$KILLRVIDEO_ADMIN_USERNAME + admin_password=$KILLRVIDEO_ADMIN_PASSWORD +fi +# If requested, create a new standard user +if [ "$KILLRVIDEO_CREATE_DSE_USER" = 'true' ]; then + # Create user and grant permission to create keyspaces (generator and web will need) + echo "=> Creating user $KILLRVIDEO_DSE_USERNAME and granting keyspace creation permissions" + cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "CREATE ROLE $KILLRVIDEO_DSE_USERNAME with LOGIN = true and PASSWORD = '$KILLRVIDEO_DSE_PASSWORD'" + echo '=> Granting keyspace creation permissions' + cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "GRANT CREATE on ALL KEYSPACES to $KILLRVIDEO_DSE_USERNAME" + cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "GRANT ALL PERMISSIONS on ALL SEARCH INDICES to $KILLRVIDEO_DSE_USERNAME" fi + +# Use the provided username/password for subsequent non-admin operations +if [ ! -z "$KILLRVIDEO_DSE_USERNAME" ]; then + dse_user=$KILLRVIDEO_DSE_USERNAME + dse_password=$KILLRVIDEO_DSE_PASSWORD +fi + +# Create the keyspace if necessary +echo '=> Ensuring keyspace is created' +keyspace_file='/opt/killrvideo-data/keyspace.cql' +if [ ! -z "$KILLRVIDEO_CASSANDRA_REPLICATION" ]; then + # TODO: check for valid replication format? https://stackoverflow.com/questions/21112707/check-if-a-string-matches-a-regex-in-bash-script + sed -i "s/{.*}/$KILLRVIDEO_CASSANDRA_REPLICATION/;" $keyspace_file +fi +cqlsh $dse_ip 9042 -f $keyspace_file -u $dse_user -p $dse_password $cql_options + +# TODO: Complete nodesync section once documentation is available +# Once we create the keyspace enable nodesync +# Commenting this out for now until we can get the correct +# documentation needed for using nodesync over SSL +#echo '=> Enabling NodeSync for KillrVideo keyspace' +#/opt/dse/resources/cassandra/bin/nodesync -cu $dse_user -cp $dse_password -h $dse_ip --cql-ssl enable -v -k killrvideo "*" + +# Create the schema if necessary +echo '=> Ensuring schema is created' +cqlsh $dse_ip 9042 -f /opt/killrvideo-data/schema.cql -k killrvideo -u $dse_user -p $dse_password $cql_options + +# Create DSE Search core if necessary +echo '=> Ensuring DSE Search is configured' +# TODO: temp workaround - if search index already exists, ALTER statements will cause non-zero exit +set +e +cqlsh $dse_ip 9042 -f /opt/killrvideo-data/videos_search.cql -k killrvideo -u $dse_user -p $dse_password $cql_options +# TODO: remove workaround +set -e + +# Wait for port 8182 (Gremlin) to be ready for up to 120 seconds +echo '=> Waiting for DSE Graph to become available' +/wait-for-it.sh -t 120 $dse_ip:8182 +echo '=> DSE Graph is available' + +# Update the gremlin-console remote.yaml file to set the remote hosts, username, and password +# This is required because the "dse gremlin-console" command does not accept username/password via command line +echo '=> Setting up remote.yaml for gremlin-console' +sed -i "s/.*hosts:.*/hosts: [$dse_ip]/;s/.*username:.*/username: $dse_user/;s/.*password:.*/password: $dse_password/;s|enableSsl:.*|enableSsl: $dse_enable_ssl, trustCertChainFile: $dse_ssl_certfile,|;" /opt/dse/resources/graph/gremlin-console/conf/remote.yaml + +# Create the graph if necessary +echo '=> Ensuring graph is created' +graph_file='/opt/killrvideo-data/killrvideo_video_recommendations_schema.groovy' +if [ ! -z "$KILLRVIDEO_GRAPH_REPLICATION" ]; then + sed -i "s/{.*}/$KILLRVIDEO_GRAPH_REPLICATION/;" $graph_file +fi +dse gremlin-console -e $graph_file + +# Register services in ETCD once all the schema are configured, using an IP that will be accessible +# internally and externally to the Docker environment +hostname="$(hostname)" + +echo '=> Registering DSE DB (Cassandra) cluster in ETCD' +curl "http://etcd:2379/v2/keys/killrvideo/services/cassandra/$hostname" -XPUT -d value="$dse_external_ip:9042" + +echo '=> Registering DSE Search in ETCD' +curl "http://etcd:2379/v2/keys/killrvideo/services/dse-search/$hostname" -XPUT -d value="$dse_external_ip:8983" + +echo '=> Registering DSE Graph in ETCD' +curl "http://etcd:2379/v2/keys/killrvideo/services/gremlin/$hostname" -XPUT -d value="$dse_external_ip:8182" + +# Don't bootstrap next time we start +echo '=> Configuration of DSE users and schema complete' +touch killrvideo_bootstrapped From 322dbe5adfd486f35bf4445852ba3bc2874b2d99 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Sun, 23 Dec 2018 22:22:40 +0100 Subject: [PATCH 2/4] Added support for KILLRVIDEO_SERVICE_DISCOVERY_DISABLED option (skip ETCD) --- bootstrap.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 2a3b13a..80dde38 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -145,14 +145,18 @@ dse gremlin-console -e $graph_file # internally and externally to the Docker environment hostname="$(hostname)" -echo '=> Registering DSE DB (Cassandra) cluster in ETCD' -curl "http://etcd:2379/v2/keys/killrvideo/services/cassandra/$hostname" -XPUT -d value="$dse_external_ip:9042" +if [ ! -z "$KILLRVIDEO_SERVICE_DISCOVERY_DISABLED" ]; then + echo '=> Skipping registration in ETCD as service discovery is disabled' +else + echo '=> Registering DSE DB (Cassandra) cluster in ETCD' + curl "http://etcd:2379/v2/keys/killrvideo/services/cassandra/$hostname" -XPUT -d value="$dse_external_ip:9042" -echo '=> Registering DSE Search in ETCD' -curl "http://etcd:2379/v2/keys/killrvideo/services/dse-search/$hostname" -XPUT -d value="$dse_external_ip:8983" + echo '=> Registering DSE Search in ETCD' + curl "http://etcd:2379/v2/keys/killrvideo/services/dse-search/$hostname" -XPUT -d value="$dse_external_ip:8983" -echo '=> Registering DSE Graph in ETCD' -curl "http://etcd:2379/v2/keys/killrvideo/services/gremlin/$hostname" -XPUT -d value="$dse_external_ip:8182" + echo '=> Registering DSE Graph in ETCD' + curl "http://etcd:2379/v2/keys/killrvideo/services/gremlin/$hostname" -XPUT -d value="$dse_external_ip:8182" +fi # Don't bootstrap next time we start echo '=> Configuration of DSE users and schema complete' From 82f98f49532efbb90e5f6444d5ff6d79afa3a274 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Wed, 2 Jan 2019 19:39:50 +0100 Subject: [PATCH 3/4] Hotfix (Missed exit statement) --- bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap.sh b/bootstrap.sh index 80dde38..e3c7142 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -6,6 +6,7 @@ echo '===> DSE Configuration' # See if we've already completed bootstrapping if [ -f killrvideo_bootstrapped ]; then echo '=> Configuration already completed, exiting' + fi # Default addresses to use for DSE cluster if starting in Docker From 496936e49b4573b28eb428c46737a3925b169fdc Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Wed, 2 Jan 2019 19:41:45 +0100 Subject: [PATCH 4/4] Hotfix (missed exit statement) --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index e3c7142..ed423dd 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -6,7 +6,7 @@ echo '===> DSE Configuration' # See if we've already completed bootstrapping if [ -f killrvideo_bootstrapped ]; then echo '=> Configuration already completed, exiting' - + exit 0 fi # Default addresses to use for DSE cluster if starting in Docker