Merge tag 'v4.21.2' of https://github.com/safe-global/safe-transactio…

…n-service into rootstock-stg

.env.test

@@ -11,3 +11,4 @@ ETHEREUM_NODE_URL=http://localhost:8545
 ETHEREUM_TRACING_NODE_URL=http://localhost:8545
 ETH_HASH_BACKEND=pysha3
 ENABLE_ANALYTICS=True
+EVENTS_QUEUE_URL=amqp://guest:guest@localhost:5672/

.github/workflows/python.yml

@@ -42,7 +42,7 @@ jobs:
         ports:
           - 6379:6379
       postgres:
-        image: postgres:13
+        image: postgres:14
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -53,6 +53,15 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
+      rabbitmq:
+        image: rabbitmq:alpine
+        options: >-
+          --health-cmd "rabbitmqctl await_startup"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - "5672:5672"
     steps:
     - name: Setup and run ganache
       run: |
@@ -86,6 +95,7 @@ jobs:
         ETHEREUM_TRACING_NODE_URL: http://localhost:8545
         ETH_HASH_BACKEND: pysha3
         REDIS_URL: redis://localhost:6379/0
+        EVENTS_QUEUE_URL: amqp://guest:guest@localhost:5672/
     - name: Send results to coveralls
       continue-on-error: true  # Ignore coveralls problems
       run: coveralls --service=github

README.md

@@ -142,11 +142,27 @@ docker exec -it safe-transaction-service-web-1 python manage.py createsuperuser
 - [v1.3.0 L2](https://github.com/safe-global/safe-deployments/blob/main/src/assets/v1.3.0/gnosis_safe_l2.json)
 - [Other related contracts and previous Safe versions](https://github.com/safe-global/safe-deployments/blob/main/src/assets)
 
-## Troubleshooting
+## Service maintenance
 
-### Issues installing grpc on a Mac M1
+Service can run into some issues when running in production:
 
-If you face issues installing the `grpc` dependency locally (required by this project) on a M1 chip, set `GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1` and `GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1` and then try to install the dependency again.
+### Indexing issues
+You can tell there are indexing issues if:
+- Executed transactions are missing from the API (`all-transactions`, `multisig-transactions`, `module-transactions`... endpoints). If you use the [Safe{Wallet} Web client](https://github.com/safe-global/safe-wallet-web) you should check what is the current state of the Safe Client Gateway cache as it might have outdated data.
+- Asset transfers (ERC20/721) are missing from `all-transactions` or `transfers` endpoints.
+- You see error logs such as "Cannot remove owner" or similar inconsistent errors when `worker-indexer` is processing decoded data.
+
+There are multiple options for this. Connect to either `web` or `worker` instances. Running commands inside of `tmux` is recommended
+(installed by default):
+- `python manage.py check_index_problems`: it will try to automatically fix missing transactions.
+Tokens related transactions (ERC20/721) will not be fixed with this method. This method will take a while, as it needs to compare
+database data with blockchain data for every Safe.
+- `python manage.py reindex_master_copies --from-block-number X --addresses 0x111 0x222`: if you know the first problematic block,
+it's faster if you trigger a manual reindex. `--addresses` argument is optional, but if you know the problematic Safes providing
+them will make reindexing **way** faster, as only those Safes will be reindexed (instead of the entire collection).
+
+If you see ERC20/ERC721 transfers missing:
+- `python manage.py reindex_erc20 --from-block-number X --addresses 0x111 0x222`: same logic as with `reindex_master_copies`.
 
 ## FAQ
 ### Why `/v1/safes/{address}` endpoint shows a nonce that indicates that a transaction was executed but the transaction is not shown or marked as executed in the other endpoints?
@@ -176,5 +192,11 @@ https://docs.safe.global/learn/safe-core/safe-core-api/available-services
 ### What means banned field in SafeContract model?
 The `banned` field in the `SafeContract` model is used to prevent indexing of certain Safes that have an unsupported `MasterCopy` or unverified proxies that have issues during indexing. This field does not remove the banned Safe and indexing can be resumed once the issue has been resolved.
 
+## Troubleshooting
+
+### Issues installing grpc on a Mac M1
+
+If you face issues installing the `grpc` dependency locally (required by this project) on a M1 chip, set `GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1` and `GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1` and then try to install the dependency again.
+
 ## Contributors
 [See contributors](https://github.com/safe-global/safe-transaction-service/graphs/contributors)

config/settings/base.py

@@ -101,6 +101,7 @@
     "safe_transaction_service.notifications.apps.NotificationsConfig",
     "safe_transaction_service.safe_messages.apps.SafeMessagesConfig",
     "safe_transaction_service.tokens.apps.TokensConfig",
+    "safe_transaction_service.events.apps.EventsConfig",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -244,6 +245,10 @@
             "safe_transaction_service.history.tasks.send_webhook_task",
             {"queue": "webhooks", "delivery_mode": "transient"},
         ),
+        (
+            "safe_transaction_service.events.tasks.send_event_to_queue_task",
+            {"queue": "webhooks", "delivery_mode": "transient"},
+        ),
         (
             "safe_transaction_service.history.tasks.reindex_mastercopies_last_hours_task",
             {"queue": "indexing"},
@@ -410,7 +415,7 @@
     "ETH_INTERNAL_TXS_BLOCK_PROCESS_LIMIT", default=10_000
 )
 ETH_INTERNAL_TXS_BLOCKS_TO_REINDEX_AGAIN = env.int(
-    "ETH_INTERNAL_TXS_BLOCKS_TO_REINDEX_AGAIN", default=6
+    "ETH_INTERNAL_TXS_BLOCKS_TO_REINDEX_AGAIN", default=10
 )
 ETH_INTERNAL_TXS_NUMBER_TRACE_BLOCKS = env.int(
     "ETH_INTERNAL_TXS_NUMBER_TRACE_BLOCKS", default=10
@@ -437,7 +442,7 @@
     "ETH_EVENTS_BLOCK_PROCESS_LIMIT_MAX", default=0
 )  # Maximum number of blocks to process together when searching for events. 0 == no limit.
 ETH_EVENTS_BLOCKS_TO_REINDEX_AGAIN = env.int(
-    "ETH_EVENTS_BLOCKS_TO_REINDEX_AGAIN", default=10
+    "ETH_EVENTS_BLOCKS_TO_REINDEX_AGAIN", default=20
 )  # Blocks to reindex again every indexer run when service is synced. Useful for RPCs not reliable
 ETH_EVENTS_GET_LOGS_CONCURRENCY = env.int(
     "ETH_EVENTS_GET_LOGS_CONCURRENCY", default=20
@@ -449,7 +454,7 @@
     "ETH_EVENTS_UPDATED_BLOCK_BEHIND", default=24 * 60 * 60 // 15
 )  # Number of blocks to consider an address 'almost updated'.
 ETH_REORG_BLOCKS = env.int(
-    "ETH_REORG_BLOCKS", default=100 if ETH_L2_NETWORK else 10
+    "ETH_REORG_BLOCKS", default=200 if ETH_L2_NETWORK else 10
 )  # Number of blocks from the current block number needed to consider a block valid/stable
 
 # Tokens
@@ -491,10 +496,11 @@
         )
     )
 
-ALERT_OUT_OF_SYNC_EVENTS_THRESHOLD = env.float(
-    "ALERT_OUT_OF_SYNC_EVENTS_THRESHOLD", default=0.1
-)  # Percentage of Safes allowed to be out of sync without alerting. By default 10%
-
+# Events
+# ------------------------------------------------------------------------------
+EVENTS_QUEUE_URL = env("EVENTS_QUEUE_URL", default=None)
+EVENTS_QUEUE_ASYNC_CONNECTION = env("EVENTS_QUEUE_ASYNC_CONNECTION", default=False)
+EVENTS_QUEUE_EXCHANGE_NAME = env("EVENTS_QUEUE_EXCHANGE_NAME", default="amq.fanout")
 
 # AWS S3 https://github.com/etianen/django-s3-storage
 # ------------------------------------------------------------------------------

config/settings/test.py

@@ -47,3 +47,5 @@
         "level": "DEBUG",
     }
 }
+
+EVENTS_QUEUE_ASYNC_CONNECTION = False

config/urls.py

@@ -14,7 +14,6 @@
         title="Safe Transaction Service API",
         default_version="v1",
         description="API to keep track of transactions sent via Gnosis Safe smart contracts",
-        contact=openapi.Contact(email="[email protected]"),
         license=openapi.License(name="MIT License"),
     ),
     validators=["flex", "ssv"],

docker-compose.yml

@@ -24,9 +24,11 @@ services:
 
   rabbitmq:
     image: rabbitmq:alpine
+    ports:
+      - "5672:5672"
 
   db:
-    image: postgres:13-alpine
+    image: postgres:14-alpine
     ports:
       - "5432:5432"
     environment:

requirements-test.txt

@@ -1,13 +1,13 @@
 -r requirements.txt
-coverage==7.2.5
-django-stubs==4.2.0
+coverage==7.2.7
+django-stubs==4.2.1
 django-test-migrations==1.3.0
 factory-boy==3.2.1
-faker==18.9.0
+faker==18.10.1
 mypy==1.0.1
-pytest==7.3.1
+pytest==7.4.0
 pytest-celery==0.0.0
 pytest-django==4.5.2
-pytest-env==0.8.1
+pytest-env==0.8.2
 pytest-rerunfailures==11.1.2
 pytest-sugar==0.9.7

requirements.txt

@@ -1,31 +1,32 @@
-boto3==1.26.142
+boto3==1.26.151
 cachetools==5.3.1
 celery==5.2.7
-django==4.2.1
+django==4.2.2
 django-cache-memoize==0.1.10
 django-celery-beat==2.5.0
 django-cors-headers==4.0.0
 django-db-geventpool==4.0.1
 django-debug-toolbar
 django-debug-toolbar-force
 django-environ==0.10.0
-django-extensions==3.2.1
+django-extensions==3.2.3
 django-filter==23.2
 django-imagekit==4.1.0
 django-model-utils==4.3.1
 django-redis==5.2.0
 django-s3-storage==0.14.0
-django-timezone-field==5.0.0
+django-timezone-field==5.1
 djangorestframework==3.14.0
 djangorestframework-camel-case==1.4.2
 docutils==0.20.1
 drf-yasg[validation]==1.21.5
-firebase-admin==6.1.0
+firebase-admin==6.2.0
 flower==1.2.0
 gunicorn[gevent]==20.1.0
 hexbytes==0.2.3
 hiredis==2.2.3
 packaging>=21.0
+pika==1.3.2
 pillow==9.5.0
 psycogreen==1.0.2
 psycopg2==2.9.6

run_tests.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 export DJANGO_SETTINGS_MODULE=config.settings.test
 export DJANGO_DOT_ENV_FILE=.env.test
-docker compose -f docker-compose.yml -f docker-compose.dev.yml build --force-rm db redis ganache
-docker compose -f docker-compose.yml -f docker-compose.dev.yml up --no-start db redis ganache
-docker compose -f docker-compose.yml -f docker-compose.dev.yml start db redis ganache
+docker compose -f docker-compose.yml -f docker-compose.dev.yml build --force-rm db redis ganache rabbitmq
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up --no-start db redis ganache rabbitmq
+docker compose -f docker-compose.yml -f docker-compose.dev.yml start db redis ganache rabbitmq
 
 sleep 10
 

safe_transaction_service/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.20.3"
+__version__ = "4.21.2"
 __version_info__ = tuple(
     int(num) if num.isdigit() else num
     for num in __version__.replace("-", ".", 1).split(".")

safe_transaction_service/contracts/tx_decoder.py

@@ -539,8 +539,10 @@ def get_contract_abi(
         :param address: Contract address
         :return: Dictionary of function selects with ABIFunction if found, `None` otherwise
         """
-        abis = ContractAbi.objects.filter(contracts__address=address).values_list(
-            "abi", flat=True
+        abis = (
+            ContractAbi.objects.filter(contracts__address=address)
+            .order_by("relevance")
+            .values_list("abi", flat=True)
         )
         if abis:
             return self._generate_selectors_with_abis_from_abi(abis[0])
@@ -564,7 +566,7 @@ def get_abi_function(
                     and selector in contract_selectors_with_abis
                 ):
                     # If the selector is available in the abi specific for the address we will use that one
-                    # Otherwise we fallback to the general abi that matches the selector
+                    # Otherwise we fall back to the general abi that matches the selector
                     return contract_selectors_with_abis[selector]
             return self.fn_selectors_with_abis[selector]
 

safe_transaction_service/events/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class EventsConfig(AppConfig):
+    name = "safe_transaction_service.events"
+    verbose_name = "Events queue for Safe Transaction Service"