diff --git a/.gitignore b/.gitignore index e2bd815..24b0214 100755 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,6 @@ dmypy.json # Pyre type checker .pyre/ - +/release/.env +/release/.env.example +/release/*.zip \ No newline at end of file diff --git a/Makefile b/Makefile index 852bdd7..803bf4c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ #!make +.PHONY: all release + DC_CONFIGS = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.yml -f seosnap-cachewarmer/docker-compose.yml -f seosnap-dashboard/docker-compose.yml DC_CONFIGS_DEV = -f docker-compose.yml -f seosnap-cacheserver/docker-compose.dev.yml -f seosnap-cachewarmer/docker-compose.dev.yml -f seosnap-dashboard/docker-compose.dev.yml @@ -37,3 +39,10 @@ develop: git submodule foreach --recursive git fetch origin develop git submodule foreach --recursive git checkout develop echo "Everything is now up to date" + + +release: + python dev/scripts/release_config.py --configs docker-compose.yml seosnap-cacheserver/docker-compose.yml seosnap-cachewarmer/docker-compose.yml seosnap-dashboard/docker-compose.yml docker-compose.yml + rm -rf release/cache release/logs + rm release/release.zip + cd release && zip release.zip * .env.example \ No newline at end of file diff --git a/dev/scripts/release_config.py b/dev/scripts/release_config.py new file mode 100644 index 0000000..aa46f9b --- /dev/null +++ b/dev/scripts/release_config.py @@ -0,0 +1,75 @@ +import yaml +from compose import config +from compose.config.config import ConfigDetails, ConfigFile +from compose.config.serialize import serialize_config + +import os +import argparse +from shutil import copyfile + +parser = argparse.ArgumentParser(description='Seosnap Release Script') +parser.add_argument('--configs', type=str, nargs='+', help='Docker compose config files') +parser.add_argument('--tag', default='latest', help='Tag for the release containers') +args = parser.parse_args() + +CONFIGS = args.configs +TAG = args.tag +OUTPUT_DIR = 'release' +WORKDIR = os.path.abspath(os.curdir) + +REPLACE_MOUNTS = { + './seosnap-cacheserver/rendertron-config.json': './rendertron-config.json' +} + +configs = [] +for file in CONFIGS: + print(f'Reading file: {file}') + with open(file, 'r') as f: + configs.append(ConfigFile(None, yaml.safe_load(f.read()))) + +print('Building config') +env = config.environment.Environment() +details = ConfigDetails( + WORKDIR, + configs, + env +) +cfg = config.load( + details, + False +) + + +def relativize(path: str) -> str: + result = f'./{os.path.relpath(path, WORKDIR)}' if path and path.startswith(WORKDIR) else path + if result in REPLACE_MOUNTS: + result = REPLACE_MOUNTS[result] + return result + + +print('Preprocessing config') +for service in cfg.services: + print(f'\tPreprocessing service: {service["name"]}') + if 'build' in service: + service.pop('build') + + for i, volume in enumerate(service['volumes']): + service['volumes'][i] = volume._replace( + internal=relativize(volume.internal), + external=relativize(volume.external) + ) + + +print(f'Writing composer file') +with open(os.path.join(OUTPUT_DIR, 'docker-compose.yml'), 'w') as f: + f.write(serialize_config(cfg, None, False)) + +copyfile( + os.path.join(WORKDIR, './seosnap-cacheserver/rendertron-config.json'), + os.path.join(OUTPUT_DIR, './rendertron-config.json'), +) + +copyfile( + os.path.join(WORKDIR, './.env.example'), + os.path.join(OUTPUT_DIR, './.env.example'), +) \ No newline at end of file diff --git a/dev/scripts/requirements.txt b/dev/scripts/requirements.txt new file mode 100644 index 0000000..0c67c28 --- /dev/null +++ b/dev/scripts/requirements.txt @@ -0,0 +1 @@ +docker-compose \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 99fa3c2..2abdef7 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,6 @@ services: db: container_name: seosnap_stack_db env_file: ./.env - volumes: - - ./dev/instance/mariadb:/var/lib/mysql networks: - seosnap diff --git a/release/docker-compose.yml b/release/docker-compose.yml new file mode 100644 index 0000000..8080643 --- /dev/null +++ b/release/docker-compose.yml @@ -0,0 +1,256 @@ +networks: + rendertron_net: + driver: bridge + seosnap: + driver: bridge + seosnap_cachewarmer: + driver: bridge + seosnap_dashboard: + driver: bridge +services: + cacheserver: + container_name: seosnap_stack_cacheserver + depends_on: + rendertron: + condition: service_started + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: null + RENDERTRON_CACHE_FILE_SUFFIX: null + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: null + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: null + RENDERTRON_CACHE_LOCK_TIMEOUT: null + RENDERTRON_CACHE_RESOURCE_METHOD: null + RENDERTRON_CACHE_RESOURCE_URL: null + RENDERTRON_CACHE_ROOT: null + RENDERTRON_MOBILE_REGEX: null + TAG: latest + image: experius/seosnap-cacheserver:${TAG} + networks: + rendertron_net: {} + seosnap: {} + ports: + - published: 5000 + target: 5000 + restart: always + volumes: + - ./cache:/app/cache:rw + - ./logs:/app/logs:rw + cachewarmer: + container_name: seosnap_stack_cachewarmer + depends_on: + cacheserver: + condition: service_started + dashboard: + condition: service_started + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: null + API_PASS: null + API_URL: null + API_VER: null + CACHEWARMER_BUFFER_SIZE: null + CACHEWARMER_CACHE_SERVER_URL: null + CACHEWARMER_CONCURRENT_REQUESTS: null + CACHEWARMER_LOG_LEVEL: null + CACHEWARMER_THREADS: null + CACHEWARMER_USER_AGENT: null + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: experius/seosnap-cachewarmer:${TAG} + networks: + seosnap: {} + seosnap_cachewarmer: {} + restart: "no" + volumes: + - ./logs:/code/logs:rw + dashboard: + container_name: seosnap_stack_dashboard + depends_on: + db: + condition: service_started + environment: + ADMIN_DEBUG: null + ADMIN_EMAIL: null + ADMIN_LOG_LEVEL: null + ADMIN_NAME: null + ADMIN_PASS: null + ADMIN_SECRET: null + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: null + DB_HOST_OVERRIDE: db + DB_NAME: null + DB_PASS: null + DB_ROOT_HOST: null + DB_USER: null + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: experius/seosnap-dashboard:${TAG} + networks: + seosnap: {} + seosnap_dashboard: {} + ports: + - published: 80 + target: 80 + restart: unless-stopped + volumes: + - ./logs:/code/logs:rw + db: + container_name: seosnap_stack_db + entrypoint: + - /entrypoint.sh + - --default-authentication-plugin=mysql_native_password + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + MYSQL_DATABASE: ${DB_NAME} + MYSQL_PASSWORD: ${DB_PASS} + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_ROOT_HOST: ${DB_ROOT_HOST} + MYSQL_USER: ${DB_USER} + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: mysql + networks: + seosnap: {} + seosnap_dashboard: {} + ports: + - published: 3306 + target: 3306 + restart: unless-stopped + volumes: + - seosnap_dashboard_db:/var/lib/mysql:rw + rendertron: + container_name: snapstack_rendertron + environment: + ADMIN_DEBUG: '1' + ADMIN_EMAIL: snaptron@snaptron.nl + ADMIN_LOG_LEVEL: ERROR + ADMIN_NAME: snaptron + ADMIN_PASS: Sn@ptron1337 + ADMIN_SECRET: 5s1pTzGIVndoSyHIdTs0rRK0INZBygQPB8iDx8fLNIy7gTrret + API_NAME: snaptron + API_PASS: Sn@ptron1337 + API_URL: http://dashboard:80/docs/ + API_VER: v1 + CACHEWARMER_BUFFER_SIZE: '50' + CACHEWARMER_CACHE_SERVER_URL: http://cacheserver:5000/render + CACHEWARMER_CONCURRENT_REQUESTS: '2' + CACHEWARMER_LOG_LEVEL: ERROR + CACHEWARMER_THREADS: '2' + CACHEWARMER_USER_AGENT: Seosnap + DB_HOST: db + DB_NAME: seosnap_dashboard + DB_PASS: snaptron_db + DB_ROOT_HOST: '%' + DB_USER: snaptron_db + EXTERNAL_CACHE_SERVER_URL: '' + RENDERTRON_CACHE_DEBUG: '0' + RENDERTRON_CACHE_FILE_SUFFIX: .json + RENDERTRON_CACHE_HEADER_REQUEST_BLACKLIST: '' + RENDERTRON_CACHE_HEADER_RESPONSE_BLACKLIST: Set-Cookie,Content-Encoding,Transfer-Encoding + RENDERTRON_CACHE_LOCK_TIMEOUT: '1' + RENDERTRON_CACHE_RESOURCE_METHOD: GET + RENDERTRON_CACHE_RESOURCE_URL: http://rendertron:3000/render + RENDERTRON_CACHE_ROOT: ./cache + RENDERTRON_MOBILE_REGEX: .*Mobile.* + TAG: latest + image: egordm/rendertron:latest + networks: + rendertron_net: {} + seosnap: {} + ports: + - published: 3000 + target: 3000 + restart: always + volumes: + - ./rendertron-config.json:/app/config.json:rw +version: '3.7' +volumes: + seosnap_dashboard_db: {} diff --git a/release/install.sh b/release/install.sh new file mode 100644 index 0000000..49d5fbf --- /dev/null +++ b/release/install.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +if [ "$1" == "clean" ]; then + echo 'Cleaning installation' + sudo rm -rf ./cache +fi + +mkdir -p cache +mkdir -p logs +if [ ! -f ".env" ]; then + cp .env.example .env + echo 'Generating new secret key' + export NEW_SECRET="$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c50)"; + sed -i "s/https\:\/\/miniwebtool.com\/django-secret-key-generator\//${NEW_SECRET}/g" .env; + + echo 'Setting new admin user login' + read -p 'Username [snaptron]: ' ADMIN_NAME + read -p 'Email [snaptron@snap.tron]: ' ADMIN_EMAIL + read -sp 'Password [Sn@ptron1337]: ' ADMIN_PASS + sed -i "s/snaptron$/${ADMIN_NAME:-snaptron}/g" .env; + sed -i "s/snaptron@snap.tron/${ADMIN_EMAIL:-snaptron@snap.tron}/g" .env; + sed -i "s/Sn@ptron1337/${ADMIN_PASS:-Sn@ptron1337}/g" .env; +fi + diff --git a/release/rendertron-config.json b/release/rendertron-config.json new file mode 100644 index 0000000..c6b2956 --- /dev/null +++ b/release/rendertron-config.json @@ -0,0 +1,3 @@ +{ + "restrictedUrlPattern": ".*(\\.png|\\.jpg|\\.jpeg|\\.gif|\\.webp|\\.mp4)($|\\?)" +} diff --git a/seosnap-cacheserver b/seosnap-cacheserver index 115e496..d42e300 160000 --- a/seosnap-cacheserver +++ b/seosnap-cacheserver @@ -1 +1 @@ -Subproject commit 115e4965958f721604de2dda4de25b4b4e1806da +Subproject commit d42e30072ce29fe2b3b4505233744cd5cfa233f0 diff --git a/seosnap-cachewarmer b/seosnap-cachewarmer index aca0d2a..f31dc57 160000 --- a/seosnap-cachewarmer +++ b/seosnap-cachewarmer @@ -1 +1 @@ -Subproject commit aca0d2ab4dbd85dc67a74781406983e6f5816c05 +Subproject commit f31dc571724c037229abe18fa7bdd009af4a0eeb diff --git a/seosnap-dashboard b/seosnap-dashboard index b7830d7..d1e810a 160000 --- a/seosnap-dashboard +++ b/seosnap-dashboard @@ -1 +1 @@ -Subproject commit b7830d7a99fcfc48a072a5ce3acb81065b01f0da +Subproject commit d1e810a635fd5e64e7cf4bc5af164f4ed2d83d29