From e19983dedb9dcbd95a44adfb45424e95cffdb03c Mon Sep 17 00:00:00 2001 From: nick8319 Date: Thu, 29 Jun 2023 16:34:36 +0200 Subject: [PATCH 01/15] feat: add RSK Mainnet and Testnet support --- requirements.txt | 3 ++- .../tokens/clients/coingecko_client.py | 3 +++ safe_transaction_service/tokens/services/price_service.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1de7ca0a..cdae6727 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,5 +31,6 @@ psycogreen==1.0.2 psycopg2==2.9.6 redis==4.5.5 requests==2.31.0 -safe-eth-py[django]==5.5.0 +git+https://github.com/protofire/safe-eth-py.git@rsk#egg=safe-eth-py +#safe-eth-py[django]==5.5.0 web3==6.5.0 diff --git a/safe_transaction_service/tokens/clients/coingecko_client.py b/safe_transaction_service/tokens/clients/coingecko_client.py index c9438695..c2ee2dc4 100644 --- a/safe_transaction_service/tokens/clients/coingecko_client.py +++ b/safe_transaction_service/tokens/clients/coingecko_client.py @@ -146,3 +146,6 @@ def get_kcs_usd_price(self) -> float: def get_metis_usd_price(self) -> float: return self.get_price("metis-token") + + def get_btc_usd_price(self) -> float: + return self.get_price("bitcoin") diff --git a/safe_transaction_service/tokens/services/price_service.py b/safe_transaction_service/tokens/services/price_service.py index 1b33060e..abb0c05d 100644 --- a/safe_transaction_service/tokens/services/price_service.py +++ b/safe_transaction_service/tokens/services/price_service.py @@ -196,6 +196,9 @@ def get_kcs_usd_price(self) -> float: return self.kucoin_client.get_kcs_usd_price() except CannotGetPrice: return self.coingecko_client.get_kcs_usd_price() + + def get_btc_usd_price(self) -> float: + return self.coingecko_client.get_btc_usd_price() @cachedmethod(cache=operator.attrgetter("cache_ether_usd_price")) @cache_memoize(60 * 30, prefix="balances-get_ether_usd_price") # 30 minutes @@ -289,6 +292,11 @@ def get_native_coin_usd_price(self) -> float: EthereumNetwork.XDC_APOTHEM_NETWORK, ): return self.get_xdc_usd_price() + elif self.ethereum_network in ( + EthereumNetwork.RSK_TESTNET, + EthereumNetwork.RSK_MAINNET, + ): + return self.get_btc_usd_price() else: return self.get_ether_usd_price() From e77d51269c6453b85fbd1e91b02546b440ff1258 Mon Sep 17 00:00:00 2001 From: Nikita Zasimuk Date: Mon, 31 Jul 2023 14:38:09 +0300 Subject: [PATCH 02/15] refactor: drop dependabot config --- .github/dependabot.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 52f8d889..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 2 -updates: - - package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - day: monday - reviewers: - - "uxio0" - - - package-ecosystem: docker - directory: "/docker/web" - schedule: - interval: weekly - day: monday - reviewers: - - "uxio0" - - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - day: monday - reviewers: - - "uxio0" From 552a32b69393a7abfebb8967ddb4cdc39b98ece1 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Tue, 10 Oct 2023 17:43:53 +0300 Subject: [PATCH 03/15] Add github ci files --- .github/workflows/ci-properties.json | 19 +++ .github/workflows/cla.yml | 36 ------ .github/workflows/deployment.yml | 39 ++++++ .github/workflows/python.yml | 181 --------------------------- 4 files changed, 58 insertions(+), 217 deletions(-) create mode 100644 .github/workflows/ci-properties.json delete mode 100644 .github/workflows/cla.yml create mode 100644 .github/workflows/deployment.yml delete mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/ci-properties.json b/.github/workflows/ci-properties.json new file mode 100644 index 00000000..4e0c5b41 --- /dev/null +++ b/.github/workflows/ci-properties.json @@ -0,0 +1,19 @@ +{ + "app_name": "rootstock-transaction-service", + "aws_region": "us-east-2", + "k8s_cluster_name": "k8-mainnet", + "registry": "docker.io", + "image_name": "sovryn/rootstock-transaction-service", + "prod_branch": "rootstock", + "dev_branch": "rootstock-stg", + "dockerfile_path": "./docker/web", + "APP_ENV_VARS": { + "PYTHONPATH": "/app/", + "C_FORCE_ROOT": "true", + "DEBUG": "0", + "ETH_L2_NETWORK": "1", + "ETH_INTERNAL_NO_FILTER": "1", + "DJANGO_SU_NAME": "sovryn", + "DJANGO_SU_EMAIL": "sovryn@sovryn.app" + } +} diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml deleted file mode 100644 index fe6cb2d1..00000000 --- a/.github/workflows/cla.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "CLA Assistant" -on: - issue_comment: - types: [ created ] - pull_request_target: - types: [ opened,closed,synchronize ] - -jobs: - CLAssistant: - runs-on: ubuntu-latest - steps: - - name: "CLA Assistant" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - # Beta Release - uses: cla-assistant/github-action@v2.3.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # the below token should have repo scope and must be manually added by you in the repository's secret - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - with: - path-to-signatures: 'signatures/version1/cla.json' - path-to-document: 'https://safe.global/cla/' - # branch should not be protected - branch: 'cla-signatures' - allowlist: hectorgomezv,moisses89,luarx,fmrsabino,luarx,rmeissner,Uxio0,*bot # may need to update this expression if we add new bots - - #below are the optional inputs - If the optional inputs are not given, then default values will be taken - #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository) - #remote-repository-name: enter the remote repository name where the signatures should be stored (Default is storing the signatures in the same repository) - #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' - #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo' - #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' - #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' - #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' - #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) - #use-dco-flag: true - If you are using DCO instead of CLA diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 00000000..ca8c58a5 --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,39 @@ +name: CI Template + +on: + push: + branches: [ rootstock, rootstock-stg ] + workflow_dispatch: + +jobs: + call-workflow-init: + uses: DistributedCollective/.github/.github/workflows/init.yml@ci_envs + with: + ref: ${{ github.ref }} + base_ref: ${{ github.base_ref }} + call-workflow-docker-build: + uses: DistributedCollective/.github/.github/workflows/docker.yml@master + needs: [call-workflow-init] + with: + KUBE_NAMESPACE: ${{ needs.call-workflow-init.outputs.KUBE_NAMESPACE }} + IMAGE_NAME: ${{ needs.call-workflow-init.outputs.image_name }} + event_name: ${{ github.event_name }} + registry: ${{ needs.call-workflow-init.outputs.registry }} + dockerfile_path: ${{ needs.call-workflow-init.outputs.dockerfile_path }} + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + call-workflow-k8s-deploy: + uses: DistributedCollective/.github/.github/workflows/deploy-k8s.yml@ci_envs + needs: [call-workflow-init, call-workflow-docker-build] + if: github.event_name != 'pull_request' + with: + KUBE_NAMESPACE: ${{ needs.call-workflow-init.outputs.KUBE_NAMESPACE }} + IMAGE_NAME: ${{ needs.call-workflow-init.outputs.image_name }} + ci_env: ${{ needs.call-workflow-init.outputs.ci_env }} + aws_region: ${{ needs.call-workflow-init.outputs.aws_region }} + k8s_cluster_name: ${{ needs.call-workflow-init.outputs.k8s_cluster_name }} + app_name: ${{ needs.call-workflow-init.outputs.app_name }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index e7339fd0..00000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,181 +0,0 @@ -name: Python CI -on: - push: - branches: - - master - - develop - pull_request: - release: - types: [ released ] - -jobs: - linting: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: pip install pre-commit - - name: Run pre-commit - run: pre-commit run --all-files - - test-app: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - services: - redis: - image: redis - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - postgres: - image: postgres:14 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - rabbitmq: - image: rabbitmq:alpine - options: >- - --health-cmd "rabbitmqctl await_startup" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - "5672:5672" - steps: - - name: Setup and run ganache - run: | - docker run --detach --publish 8545:8545 --network-alias ganache -e DOCKER=true trufflesuite/ganache:latest --defaultBalanceEther 10000 --gasLimit 10000000 -a 30 --chain.chainId 1337 --chain.networkId 1337 -d - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: 'requirements*.txt' - - name: Install dependencies - run: | - pip install wheel - pip install -r requirements-test.txt coveralls - env: - PIP_USE_MIRRORS: true - - name: Run tests and coverage - run: | - python manage.py check - python manage.py makemigrations --check --dry-run - coverage run --source=$SOURCE_FOLDER -m pytest -rxXs --reruns 3 - env: - SOURCE_FOLDER: safe_transaction_service - CELERY_BROKER_URL: redis://localhost:6379/0 - COINMARKETCAP_API_TOKEN: ${{ secrets.COINMARKETCAP_API_TOKEN }} - DATABASE_URL: psql://postgres:postgres@localhost/postgres - DJANGO_SETTINGS_MODULE: config.settings.test - ETHEREUM_MAINNET_NODE: ${{ secrets.ETHEREUM_MAINNET_NODE }} - ETHEREUM_NODE_URL: http://localhost:8545 - ETHEREUM_TRACING_NODE_URL: http://localhost:8545 - ETH_HASH_BACKEND: pysha3 - REDIS_URL: redis://localhost:6379/0 - EVENTS_QUEUE_URL: amqp://guest:guest@localhost:5672/ - - name: Send results to coveralls - continue-on-error: true # Ignore coveralls problems - run: coveralls --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Required for coveralls - docker-deploy: - runs-on: ubuntu-latest - needs: - - linting - - test-app - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || (github.event_name == 'release' && github.event.action == 'released') - steps: - - uses: actions/checkout@v3 - - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - uses: docker/setup-buildx-action@v2 - - name: Dockerhub login - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Deploy Master - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v4 - with: - context: . - file: docker/web/Dockerfile - push: true - tags: safeglobal/safe-transaction-service:staging - platforms: | - linux/amd64 - linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Deploy Develop - if: github.ref == 'refs/heads/develop' - uses: docker/build-push-action@v4 - with: - context: . - file: docker/web/Dockerfile - push: true - tags: safeglobal/safe-transaction-service:develop - platforms: | - linux/amd64 - linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Deploy Tag - if: (github.event_name == 'release' && github.event.action == 'released') - uses: docker/build-push-action@v4 - with: - context: . - file: docker/web/Dockerfile - push: true - tags: | - safeglobal/safe-transaction-service:${{ github.event.release.tag_name }} - safeglobal/safe-transaction-service:latest - platforms: | - linux/amd64 - linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - autodeploy: - runs-on: ubuntu-latest - needs: [docker-deploy] - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' - steps: - - uses: actions/checkout@v3 - - name: Deploy Staging - if: github.ref == 'refs/heads/master' - run: bash scripts/autodeploy.sh - env: - AUTODEPLOY_URL: ${{ secrets.AUTODEPLOY_URL }} - AUTODEPLOY_TOKEN: ${{ secrets.AUTODEPLOY_TOKEN }} - TARGET_ENV: "staging" - - name: Deploy Develop - if: github.ref == 'refs/heads/develop' - run: bash scripts/autodeploy.sh - env: - AUTODEPLOY_URL: ${{ secrets.AUTODEPLOY_URL }} - AUTODEPLOY_TOKEN: ${{ secrets.AUTODEPLOY_TOKEN }} - TARGET_ENV: "develop" From 423a1e9cec1ba60bb4a1fafe0a1351bbe86b70f5 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Tue, 10 Oct 2023 18:00:28 +0300 Subject: [PATCH 04/15] Add deployment file --- deployment.yaml | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 deployment.yaml diff --git a/deployment.yaml b/deployment.yaml new file mode 100644 index 00000000..bc81c34a --- /dev/null +++ b/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: '1' + labels: + app: rootstock-transaction-service + name: rootstock-transaction-service +spec: + replicas: 1 + selector: + matchLabels: + app: rootstock-transaction-service + strategy: + type: Recreate + template: + metadata: + labels: + app: rootstock-transaction-service + spec: + containers: + - image: sovryn/safe-config + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + name: web + resources: {} + tty: true + volumeMounts: + - mountPath: /nginx + name: nginx-shared + - image: nginx:1.20-alpine + name: nginx + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + ports: + - containerPort: 80 + resources: {} + volumeMounts: + - name: nginxconf + mountPath: /etc/nginx/templates + - mountPath: /nginx + name: nginx-shared + restartPolicy: Always + volumes: + - name: nginx-shared + emptyDir: {} + - name: nginxconf + secret: + secretName: rootstock-transaction-service-nginx + imagePullSecrets: + - name: regcred +--- +apiVersion: v1 +kind: Service +metadata: + name: rootstock-transaction-service + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-2:768438872063:certificate/9fa02b26-0ee5-4301-bdd6-8474344885ff +spec: + selector: + app: rootstock-transaction-service + ports: + - port: 443 + targetPort: 80 + type: LoadBalancer +--- From 570663a1a87c103177ea6edfedb177b646199f6f Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 11 Oct 2023 15:21:23 +0300 Subject: [PATCH 05/15] Update config --- .github/workflows/ci-properties.json | 3 ++- deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-properties.json b/.github/workflows/ci-properties.json index 4e0c5b41..19fe9e4f 100644 --- a/.github/workflows/ci-properties.json +++ b/.github/workflows/ci-properties.json @@ -14,6 +14,7 @@ "ETH_L2_NETWORK": "1", "ETH_INTERNAL_NO_FILTER": "1", "DJANGO_SU_NAME": "sovryn", - "DJANGO_SU_EMAIL": "sovryn@sovryn.app" + "DJANGO_SU_EMAIL": "sovryn@sovryn.app", + "DJANGO_SETTINGS_MODULE": "config.settings.productddion" } } diff --git a/deployment.yaml b/deployment.yaml index bc81c34a..de35d534 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -19,7 +19,7 @@ spec: app: rootstock-transaction-service spec: containers: - - image: sovryn/safe-config + - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: - secretRef: From f326ccb601a4afb9fa00011ff0d6f5dc9e02f2d8 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 11 Oct 2023 15:54:21 +0300 Subject: [PATCH 06/15] Update deployment config --- deployment.yaml | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/deployment.yaml b/deployment.yaml index de35d534..eedf787e 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -19,6 +19,68 @@ spec: app: rootstock-transaction-service spec: containers: + - image: sovryn/rootstock-transaction-service + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + ports: + - containerPort: 5555 + env: + - name: WORKER_QUEUES + value: "default,indexing" + - name: RUN_MIGRATION + value: "1" + command: ["docker/web/celery/flower/run.sh"] + name: flower + resources: {} + tty: true + - image: sovryn/rootstock-transaction-service + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + env: + - name: WORKER_QUEUES + value: "contracts,tokens" + command: ["docker/web/celery/worker/run.sh"] + name: contracts-tokens-worker + resources: {} + tty: true + - image: sovryn/rootstock-transaction-service + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + env: + - name: WORKER_QUEUES + value: "notifications,webhooks" + command: ["docker/web/celery/worker/run.sh"] + name: notifications-webhooks-worker + resources: {} + tty: true + - image: sovryn/rootstock-transaction-service + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + env: + - name: WORKER_QUEUES + value: "default,indexing" + - name: RUN_MIGRATION + value: "1" + command: ["docker/web/celery/worker/run.sh"] + name: indexer-worker + resources: {} + tty: true - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: From 2ca8554432a5609f5f81ea890a206420edae043e Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 11 Oct 2023 15:58:55 +0300 Subject: [PATCH 07/15] Fix command tty --- deployment.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deployment.yaml b/deployment.yaml index eedf787e..7a9620be 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -36,7 +36,6 @@ spec: command: ["docker/web/celery/flower/run.sh"] name: flower resources: {} - tty: true - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: @@ -50,7 +49,6 @@ spec: command: ["docker/web/celery/worker/run.sh"] name: contracts-tokens-worker resources: {} - tty: true - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: @@ -64,7 +62,6 @@ spec: command: ["docker/web/celery/worker/run.sh"] name: notifications-webhooks-worker resources: {} - tty: true - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: @@ -80,7 +77,6 @@ spec: command: ["docker/web/celery/worker/run.sh"] name: indexer-worker resources: {} - tty: true - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: @@ -90,10 +86,10 @@ spec: name: rootstock-transaction-service-secret name: web resources: {} - tty: true volumeMounts: - mountPath: /nginx name: nginx-shared + command: ["docker/web/run_web.sh"] - image: nginx:1.20-alpine name: nginx envFrom: From e8be731bcbed7a270d4add472908c3fd6d12d85c Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 11 Oct 2023 17:00:47 +0300 Subject: [PATCH 08/15] Fix entrypoing and add scheduler --- deployment.yaml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/deployment.yaml b/deployment.yaml index 7a9620be..103dcfae 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -33,9 +33,26 @@ spec: value: "default,indexing" - name: RUN_MIGRATION value: "1" - command: ["docker/web/celery/flower/run.sh"] + args: ["docker/web/celery/flower/run.sh"] name: flower resources: {} + - image: sovryn/rootstock-transaction-service + imagePullPolicy: Always + envFrom: + - secretRef: + name: rootstock-transaction-service + - secretRef: + name: rootstock-transaction-service-secret + ports: + - containerPort: 5555 + env: + - name: WORKER_QUEUES + value: "default,indexing" + - name: RUN_MIGRATION + value: "1" + args: ["docker/web/celery/scheduler/run.sh"] + name: scheduler + resources: {} - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: @@ -46,7 +63,7 @@ spec: env: - name: WORKER_QUEUES value: "contracts,tokens" - command: ["docker/web/celery/worker/run.sh"] + args: ["docker/web/celery/worker/run.sh"] name: contracts-tokens-worker resources: {} - image: sovryn/rootstock-transaction-service @@ -59,7 +76,7 @@ spec: env: - name: WORKER_QUEUES value: "notifications,webhooks" - command: ["docker/web/celery/worker/run.sh"] + args: ["docker/web/celery/worker/run.sh"] name: notifications-webhooks-worker resources: {} - image: sovryn/rootstock-transaction-service @@ -74,7 +91,7 @@ spec: value: "default,indexing" - name: RUN_MIGRATION value: "1" - command: ["docker/web/celery/worker/run.sh"] + args: ["docker/web/celery/worker/run.sh"] name: indexer-worker resources: {} - image: sovryn/rootstock-transaction-service @@ -89,7 +106,7 @@ spec: volumeMounts: - mountPath: /nginx name: nginx-shared - command: ["docker/web/run_web.sh"] + args: ["docker/web/run_web.sh"] - image: nginx:1.20-alpine name: nginx envFrom: From dcbb7b2f4efd46548a0d7895a37a2ffc9976b076 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 11 Oct 2023 18:06:33 +0300 Subject: [PATCH 09/15] fix typo --- deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment.yaml b/deployment.yaml index 103dcfae..2756bad1 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -31,7 +31,7 @@ spec: env: - name: WORKER_QUEUES value: "default,indexing" - - name: RUN_MIGRATION + - name: RUN_MIGRATIONS value: "1" args: ["docker/web/celery/flower/run.sh"] name: flower @@ -48,7 +48,7 @@ spec: env: - name: WORKER_QUEUES value: "default,indexing" - - name: RUN_MIGRATION + - name: RUN_MIGRATIONS value: "1" args: ["docker/web/celery/scheduler/run.sh"] name: scheduler @@ -89,7 +89,7 @@ spec: env: - name: WORKER_QUEUES value: "default,indexing" - - name: RUN_MIGRATION + - name: RUN_MIGRATIONS value: "1" args: ["docker/web/celery/worker/run.sh"] name: indexer-worker From cd36f90d90b04ad190478d0a2db23f6b1ebfc5bf Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Wed, 18 Oct 2023 10:49:36 +0300 Subject: [PATCH 10/15] adjust settings --- config/settings/production.py | 1 + deployment.yaml | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/settings/production.py b/config/settings/production.py index a5c71ba0..f5aea51c 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -64,6 +64,7 @@ # ------------------------------------------------------------------------------ # Django Admin URL regex. ADMIN_URL = env("DJANGO_ADMIN_URL", default="admin/") +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTOCOL", "https") # CELERY # ------------------------------------------------------------------------------ diff --git a/deployment.yaml b/deployment.yaml index 2756bad1..56f88b0b 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -63,6 +63,8 @@ spec: env: - name: WORKER_QUEUES value: "contracts,tokens" + - name: RUN_MIGRATIONS + value: "1" args: ["docker/web/celery/worker/run.sh"] name: contracts-tokens-worker resources: {} @@ -76,6 +78,8 @@ spec: env: - name: WORKER_QUEUES value: "notifications,webhooks" + - name: RUN_MIGRATIONS + value: "1" args: ["docker/web/celery/worker/run.sh"] name: notifications-webhooks-worker resources: {} @@ -115,11 +119,12 @@ spec: - secretRef: name: rootstock-transaction-service-secret ports: - - containerPort: 80 + - containerPort: 8000 resources: {} volumeMounts: - name: nginxconf - mountPath: /etc/nginx/templates + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf - mountPath: /nginx name: nginx-shared restartPolicy: Always @@ -144,6 +149,6 @@ spec: app: rootstock-transaction-service ports: - port: 443 - targetPort: 80 + targetPort: 8000 type: LoadBalancer --- From 7967a13f3e554abd387f023cd62a034c21e7869b Mon Sep 17 00:00:00 2001 From: Renat Skitsan Date: Fri, 17 Nov 2023 14:37:45 +0200 Subject: [PATCH 11/15] Update run.sh to enable events for the worker --- docker/web/celery/worker/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/web/celery/worker/run.sh b/docker/web/celery/worker/run.sh index 5e5cffe8..9b9420d0 100755 --- a/docker/web/celery/worker/run.sh +++ b/docker/web/celery/worker/run.sh @@ -28,8 +28,10 @@ python manage.py check_chainid_matches export C_FORCE_ROOT=true echo "==> $(date +%H:%M:%S) ==> Running Celery worker with a max_memory_per_child of ${MAX_MEMORY_PER_CHILD} <==" +# https://github.com/sumitasok/celery/issues/5#issuecomment-781717855 exec celery -C -A config.celery_app worker \ --loglevel $log_level --pool=gevent \ + -E \ --concurrency=${TASK_CONCURRENCY} \ --max-memory-per-child=${MAX_MEMORY_PER_CHILD} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ From 9c80a5f62876a7afb9f617044af7473241d5edd5 Mon Sep 17 00:00:00 2001 From: Renat Skitsan Date: Fri, 17 Nov 2023 14:40:26 +0200 Subject: [PATCH 12/15] Update run.sh --- docker/web/celery/worker/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/web/celery/worker/run.sh b/docker/web/celery/worker/run.sh index 5e5cffe8..ae980c62 100755 --- a/docker/web/celery/worker/run.sh +++ b/docker/web/celery/worker/run.sh @@ -28,8 +28,10 @@ python manage.py check_chainid_matches export C_FORCE_ROOT=true echo "==> $(date +%H:%M:%S) ==> Running Celery worker with a max_memory_per_child of ${MAX_MEMORY_PER_CHILD} <==" +# https://github.com/sumitasok/celery/issues/5#issuecomment-781717855 exec celery -C -A config.celery_app worker \ --loglevel $log_level --pool=gevent \ + -E \ --concurrency=${TASK_CONCURRENCY} \ --max-memory-per-child=${MAX_MEMORY_PER_CHILD} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ From 97bd89d86c63f5e87c1b6f3753753f42a5b0f6d1 Mon Sep 17 00:00:00 2001 From: Renat Skitsan Date: Fri, 17 Nov 2023 14:44:20 +0200 Subject: [PATCH 13/15] Update run.sh --- docker/web/celery/worker/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/web/celery/worker/run.sh b/docker/web/celery/worker/run.sh index 9b9420d0..90ed129d 100755 --- a/docker/web/celery/worker/run.sh +++ b/docker/web/celery/worker/run.sh @@ -29,9 +29,8 @@ export C_FORCE_ROOT=true echo "==> $(date +%H:%M:%S) ==> Running Celery worker with a max_memory_per_child of ${MAX_MEMORY_PER_CHILD} <==" # https://github.com/sumitasok/celery/issues/5#issuecomment-781717855 -exec celery -C -A config.celery_app worker \ +exec celery -C -A config.celery_app worker --task-events \ --loglevel $log_level --pool=gevent \ - -E \ --concurrency=${TASK_CONCURRENCY} \ --max-memory-per-child=${MAX_MEMORY_PER_CHILD} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ From 0f6a81cb321d6e9a09736177964eeec7af7e8538 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Mon, 5 Feb 2024 12:50:35 +0200 Subject: [PATCH 14/15] add resources & ingress & ci update --- .github/workflows/ci-properties.json | 8 +++++++- .github/workflows/deployment.yml | 4 ++-- deployment.yaml | 6 +----- ingress.yaml | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 ingress.yaml diff --git a/.github/workflows/ci-properties.json b/.github/workflows/ci-properties.json index 19fe9e4f..a9e55f57 100644 --- a/.github/workflows/ci-properties.json +++ b/.github/workflows/ci-properties.json @@ -15,6 +15,12 @@ "ETH_INTERNAL_NO_FILTER": "1", "DJANGO_SU_NAME": "sovryn", "DJANGO_SU_EMAIL": "sovryn@sovryn.app", - "DJANGO_SETTINGS_MODULE": "config.settings.productddion" + "DJANGO_SETTINGS_MODULE": "config.settings.production" + }, + "DEV_ENV_VARS": { + "INGRESS_HOSTNAME": "safe-tx.test.sovryn.app" + }, + "PROD_ENV_VARS": { + "INGRESS_HOSTNAME": "safe-tx.sovryn.app" } } diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index ca8c58a5..a2ce9299 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -7,7 +7,7 @@ on: jobs: call-workflow-init: - uses: DistributedCollective/.github/.github/workflows/init.yml@ci_envs + uses: DistributedCollective/.github/.github/workflows/init.yml@master with: ref: ${{ github.ref }} base_ref: ${{ github.base_ref }} @@ -24,7 +24,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} call-workflow-k8s-deploy: - uses: DistributedCollective/.github/.github/workflows/deploy-k8s.yml@ci_envs + uses: DistributedCollective/.github/.github/workflows/deploy-k8s.yml@master needs: [call-workflow-init, call-workflow-docker-build] if: github.event_name != 'pull_request' with: diff --git a/deployment.yaml b/deployment.yaml index 56f88b0b..bf4578da 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -141,14 +141,10 @@ apiVersion: v1 kind: Service metadata: name: rootstock-transaction-service - annotations: - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-2:768438872063:certificate/9fa02b26-0ee5-4301-bdd6-8474344885ff spec: selector: app: rootstock-transaction-service ports: - port: 443 targetPort: 8000 - type: LoadBalancer ---- + type: NodePort diff --git a/ingress.yaml b/ingress.yaml new file mode 100644 index 00000000..4add829e --- /dev/null +++ b/ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rootstock-transaction-service-ingress +spec: + ingressClassName: nginx + rules: + - host: ingress.hostname + http: + paths: + - backend: + service: + name: rootstock-transaction-service + port: + number: 443 + path: / + pathType: Prefix From 52e3acf1cc68eb867e669985517a348b4a30c4d1 Mon Sep 17 00:00:00 2001 From: Renat Gofman Date: Mon, 5 Feb 2024 12:52:34 +0200 Subject: [PATCH 15/15] add resources --- deployment.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment.yaml b/deployment.yaml index bf4578da..9b73d730 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -35,7 +35,10 @@ spec: value: "1" args: ["docker/web/celery/flower/run.sh"] name: flower - resources: {} + resources: + requests: + memory: "3Gi" + cpu: "200m" - image: sovryn/rootstock-transaction-service imagePullPolicy: Always envFrom: