From 8cf4c98129d8decfe3d814e104de2018e4d817e8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Oct 2019 19:37:43 +0200 Subject: [PATCH 1/6] Add Dockerfile for fast usage on future projects --- Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..960bc39 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3-alpine + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY data data/ +COPY *.py *.py.sample ./ +RUN mv dvconfig.py.sample dvconfig.py + +CMD [ "python", "./create_sample_data.py" ] From 60da48d02187a3ae4d24795b10f7c10ad322c1c2 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Oct 2019 21:33:48 +0200 Subject: [PATCH 2/6] Make config read from os environment variables and provide defaults if empty. --- dvconfig.py.sample | 47 ++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/dvconfig.py.sample b/dvconfig.py.sample index 30e157f..ae51451 100644 --- a/dvconfig.py.sample +++ b/dvconfig.py.sample @@ -1,25 +1,28 @@ -base_url = 'http://localhost:8080' -api_token = '' +import os + +base_url = os.getenv('BASE_URL', 'http://localhost:8080') +api_token = os.getenv('API_TOKEN', '') + # sample data will be created in the following order sample_data = [ -'data/dataverses/open-source-at-harvard/open-source-at-harvard.json', -'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/dataverse-project.json', -'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/datasets/dataverse-irc-metrics/dataverse-irc-metrics.json', -'data/dataverses/ecastro/ecastro.json', -'data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json', -'data/dataverses/manchester/manchester.json', -'data/dataverses/manchester/datasets/test-dataset/test-dataset.json', -'data/dataverses/HCPDS/HCPDS.json', -'data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json', -'data/dataverses/cms/cms.json', -'data/dataverses/cms/datasets/cmssampledata/cmssampledata.json', -'data/dataverses/scholcommlab/scholcommlab.json', -'data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json', -'data/dataverses/ubiquity-press/ubiquity-press.json', -'data/dataverses/ubiquity-press/dataverses/jopd/jopd.json', -'data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json', -'data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json', -'data/dataverses/open-source-at-harvard/datasets/open-source-at-harvard/open-source-at-harvard.json', -'data/dataverses/king/king.json', -'data/dataverses/king/datasets/cause-of-death/cause-of-death.json', + 'data/dataverses/open-source-at-harvard/open-source-at-harvard.json', + 'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/dataverse-project.json', + 'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/datasets/dataverse-irc-metrics/dataverse-irc-metrics.json', + 'data/dataverses/ecastro/ecastro.json', + 'data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json', + 'data/dataverses/manchester/manchester.json', + 'data/dataverses/manchester/datasets/test-dataset/test-dataset.json', + 'data/dataverses/HCPDS/HCPDS.json', + 'data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json', + 'data/dataverses/cms/cms.json', + 'data/dataverses/cms/datasets/cmssampledata/cmssampledata.json', + 'data/dataverses/scholcommlab/scholcommlab.json', + 'data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json', + 'data/dataverses/ubiquity-press/ubiquity-press.json', + 'data/dataverses/ubiquity-press/dataverses/jopd/jopd.json', + 'data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json', + 'data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json', + 'data/dataverses/open-source-at-harvard/datasets/open-source-at-harvard/open-source-at-harvard.json', + 'data/dataverses/king/king.json', + 'data/dataverses/king/datasets/cause-of-death/cause-of-death.json', ] From 9dbae8fe3716ab5d432eda4fda8865bcb4141e4d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 5 Dec 2019 15:09:52 +0100 Subject: [PATCH 3/6] Use a more sophisticated env var concatenating to autobuild the Dataverse base url in best case. --- dvconfig.py.sample | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dvconfig.py.sample b/dvconfig.py.sample index ae51451..d72ca09 100644 --- a/dvconfig.py.sample +++ b/dvconfig.py.sample @@ -1,6 +1,13 @@ import os -base_url = os.getenv('BASE_URL', 'http://localhost:8080') +# Create the base_url from different parts (very usefull on K8s) or just +# read completely from a single env var. Default to "http://localhost:8080" +host = os.getenv('DATAVERSE_SERVICE_HOST', 'localhost') +port = os.getenv('DATAVERSE_SERVICE_PORT_HTTP', '8080') +proto = os.getenv('DATAVERSE_SERVICE_PORT_PROTO', 'http') +subpath = os.getenv('DATAVERSE_SERVICE_SUBPATH', '') +base_url = os.getenv('BASE_URL', proto+'://'+host+':'+port+subpath) + api_token = os.getenv('API_TOKEN', '') # sample data will be created in the following order From 17a199e544c17e3cde010923ba5a76aaf520303d Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 5 Dec 2019 16:41:10 +0100 Subject: [PATCH 4/6] Refactor get_api_token.py to be K8s and docker friendly. --- get_api_token.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/get_api_token.py b/get_api_token.py index 896a6d1..e1eb109 100644 --- a/get_api_token.py +++ b/get_api_token.py @@ -1,14 +1,33 @@ from pyDataverse.api import Api import json import dvconfig +from pathlib import Path +import os +import sys + base_url = dvconfig.base_url -api_token = dvconfig.api_token -api = Api(base_url, api_token) -username = 'dataverseAdmin' -password = 'admin1' +api = Api(base_url, '') + +username = os.getenv('DATAVERSE_USER', 'dataverseAdmin') +password = os.getenv('DATAVERSE_PASSWORD', 'admin1') +# On K8s or with Docker we should get secrets from files, not env vars +if Path(password).is_file(): + f = open(Path(password), 'r') + password = f.read().strip() + f.close() + endpoint = '/builtin-users/' + username + '/api-token' params = {} params['password'] = password -resp = api.get_request(endpoint, params=params, auth=True) -api_token = resp.json()['data']['message'] -print(api_token) + +resp = api.get_request(endpoint, params=params, auth=False) +if resp.json()['status'] == "OK": + api_token = resp.json()['data']['message'] + print(api_token) + sys.exit(0) +else: + print("ERROR receiving API token:", file=sys.stderr) + print(resp.json(), file=sys.stderr) + print("Did you enable :AllowApiTokenLookupViaApi configuration option?", file=sys.stderr) + print("See http://guides.dataverse.org/en/latest/installation/config.html#allowapitokenlookupviaapi", file=sys.stderr) + sys.exit(1) From f0f9cc5115092d992fc143499959974ac84aa9fe Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 5 Dec 2019 16:58:44 +0100 Subject: [PATCH 5/6] Adding docs about get_api_token.py --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 0f8a223..a2f1f71 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,36 @@ All of the steps above can be automated on an fresh installation of Dataverse on For more information on spinning up Dataverse on AWS (especially if you don't have the `aws` executable installed), see http://guides.dataverse.org/en/latest/developers/deployment.html +## Usage in automated processes without API key +Sometimes you don't want to retrieve an API key manually, like at demo times +or when you do automatic deployments of sample data. + +For these cases, a script `get_api_token.py` has been added for your convenience. + +You can use it like follows (just an example): +``` +API_TOKEN=`python get_api_token.py` python create_sample_data.py +``` + +The script understands two additional environment variables (in addition to +those from `dvconfig.py`): +* `DATAVERSE_USER` + * Username of the user whos API token we want to retrieve + * Defaults to `dataverseAdmin` +* `DATAVERSE_PASSWORD` + * either a cleartext password or a path to a file containing the clear text + password for the user. + * Defaults to `admin1` (usable for dataverse-ansible and dataverse-kubernetes) + +In case of a successfull retrieval, it will print the password to standard out. +Error will be written to standard error, so you will see it at the top of +a failed attempt to load the data. + +Please be aware that you will have to enable +[:AllowApiTokenLookupViaApi](http://guides.dataverse.org/en/latest/installation/config.html#allowapitokenlookupviaapi) +configuration option in your Dataverse to use this script. You can disable +after deploying the data, no harm done. + ## Contributing We love contributors! Please see our [Contributing Guide][] for ways you can help. From 4ccd3d5fab7b4110a63b827e10f243cc973fe648 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 6 Feb 2020 12:34:16 +0100 Subject: [PATCH 6/6] Add Jenkinsfile for container image building and pushing to DockerHub --- Jenkinsfile | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..f975d1e --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,59 @@ +void setBuildStatus(String message, String state) { + step([ + $class: "GitHubCommitStatusSetter", + reposSource: [$class: "ManuallyEnteredRepositorySource", url: "${env.GIT_URL}"], + contextSource: [$class: "ManuallyEnteredCommitContextSource", context: "ci/docker/dataverse-sample-data"], + errorHandlers: [[$class: "ChangingBuildStatusErrorHandler", result: "UNSTABLE"]], + statusResultSource: [ $class: "ConditionalStatusResultSource", results: [[$class: "AnyBuildResult", message: message, state: state]] ] + ]); +} + +pipeline { + agent any + environment { + DOCKER_IMAGE_NAME = "iqss/dataverse-sample-data" + DOCKER_IMAGE_TAG = "build-${env.BRANCH_NAME}" + DOCKER_WORKDIR = "." + DOCKER_HUB_CRED = "dockerhub-dataversebot" + DOCKER_REGISTRY = "https://registry.hub.docker.com" + } + stages { + stage('build') { + when { + anyOf { + branch 'master' + branch 'PR-14' + } + } + steps { + script { + docker_image = docker.build("${env.DOCKER_IMAGE_NAME}:${env.DOCKER_IMAGE_TAG}", "--pull ${env.DOCKER_WORKDIR}") + } + } + } + stage('push') { + when { + anyOf { + branch 'master' + branch 'PR-14' + } + } + steps { + script { + // Push master image to latest tag + docker.withRegistry("${env.DOCKER_REGISTRY}", "${env.DOCKER_HUB_CRED}") { + docker_image.push("latest") + } + } + } + } + } + post { + success { + setBuildStatus("Image build and push succeeded", "SUCCESS"); + } + failure { + setBuildStatus("Image build or push failed", "FAILURE"); + } + } +}