From d40c1f58b0b450b0fe287761f4f42d1edda8bf8a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Thu, 11 Jul 2024 00:57:22 +0200 Subject: [PATCH] style: hatch run linting:fmt + manual fixes --- deployment/migrations/env.py | 2 +- .../0007_0bfde82697c8_balance_views.py | 1 - .../0008_eef5f95853bf_trusted_messages.py | 1 - .../0009_8edf69c47884_file_pin_owner_index.py | 1 - .../0010_8a5eaab15d40_address_stats_view.py | 1 - .../0013_7ab62bd0a3b1_messages_post_index.py | 1 - ...a92b500049_message_content_in_file_pins.py | 1 - .../0016_77e68941d36c_fix_ipfs_files_size.py | 1 - .../0018_7bcb8e5fe186_fix_vm_cost_view.py | 1 - .../0021_08602db6c78f_ccn_metric_view.py | 1 - .../0022_e682fc8f9506_fix_vm_costs_view.py | 1 - pyproject.toml | 2 +- src/aleph/__init__.py | 2 +- src/aleph/api_entrypoint.py | 13 +- src/aleph/chains/abc.py | 10 +- src/aleph/chains/avalanche.py | 3 +- src/aleph/chains/chain_data_service.py | 15 +-- src/aleph/chains/common.py | 4 +- src/aleph/chains/connector.py | 1 + src/aleph/chains/cosmos.py | 1 + src/aleph/chains/ethereum.py | 3 +- src/aleph/chains/indexer_reader.py | 18 +-- src/aleph/chains/nuls.py | 12 +- src/aleph/chains/nuls2.py | 11 +- src/aleph/chains/nuls_aleph_sdk.py | 2 +- src/aleph/chains/signature_verifier.py | 2 +- src/aleph/chains/solana.py | 1 + src/aleph/chains/substrate.py | 1 + src/aleph/chains/tezos.py | 12 +- src/aleph/commands.py | 4 +- src/aleph/db/accessors/aggregates.py | 32 ++--- src/aleph/db/accessors/balances.py | 4 +- src/aleph/db/accessors/chains.py | 7 +- src/aleph/db/accessors/cost.py | 5 +- src/aleph/db/accessors/files.py | 11 +- src/aleph/db/accessors/messages.py | 49 ++++--- src/aleph/db/accessors/peers.py | 5 +- src/aleph/db/accessors/pending_messages.py | 10 +- src/aleph/db/accessors/pending_txs.py | 6 +- src/aleph/db/accessors/posts.py | 31 ++--- src/aleph/db/accessors/vms.py | 13 +- src/aleph/db/connection.py | 6 +- src/aleph/db/models/__init__.py | 23 ++-- src/aleph/db/models/aggregates.py | 2 +- src/aleph/db/models/balances.py | 2 +- src/aleph/db/models/base.py | 4 +- src/aleph/db/models/chains.py | 11 +- src/aleph/db/models/files.py | 26 ++-- src/aleph/db/models/messages.py | 21 +-- src/aleph/db/models/peers.py | 2 +- src/aleph/db/models/pending_messages.py | 15 ++- src/aleph/db/models/posts.py | 3 +- src/aleph/db/models/vms.py | 11 +- src/aleph/exceptions.py | 15 +-- src/aleph/handlers/content/aggregate.py | 17 +-- src/aleph/handlers/content/forget.py | 26 ++-- src/aleph/handlers/content/post.py | 15 ++- src/aleph/handlers/content/store.py | 18 +-- src/aleph/handlers/content/vm.py | 56 ++++---- src/aleph/handlers/message_handler.py | 18 +-- src/aleph/jobs/__init__.py | 10 +- src/aleph/jobs/fetch_pending_messages.py | 24 ++-- src/aleph/jobs/job_utils.py | 5 +- src/aleph/jobs/process_pending_messages.py | 22 ++-- src/aleph/jobs/process_pending_txs.py | 15 ++- src/aleph/network.py | 4 +- src/aleph/schemas/api/accounts.py | 5 +- src/aleph/schemas/api/messages.py | 37 +++--- src/aleph/schemas/base_messages.py | 5 +- src/aleph/schemas/chains/indexer_response.py | 14 +- src/aleph/schemas/chains/sync_events.py | 6 +- .../schemas/chains/tezos_indexer_response.py | 4 +- src/aleph/schemas/message_confirmation.py | 8 +- src/aleph/schemas/pending_messages.py | 12 +- .../services/cache/materialized_views.py | 2 +- src/aleph/services/cache/node_cache.py | 4 +- src/aleph/services/cost.py | 5 +- src/aleph/services/ipfs/common.py | 1 - src/aleph/services/ipfs/pubsub.py | 3 +- src/aleph/services/ipfs/service.py | 8 +- src/aleph/services/p2p/__init__.py | 3 +- src/aleph/services/p2p/http.py | 2 +- src/aleph/services/p2p/jobs.py | 9 +- src/aleph/services/p2p/peers.py | 1 + src/aleph/services/p2p/protocol.py | 2 +- src/aleph/services/peers/monitor.py | 4 +- src/aleph/services/storage/engine.py | 13 +- .../services/storage/garbage_collector.py | 11 +- src/aleph/services/utils.py | 2 +- src/aleph/storage.py | 11 +- src/aleph/toolkit/exceptions.py | 2 +- src/aleph/toolkit/json.py | 2 +- src/aleph/toolkit/libp2p_stubs/crypto/keys.py | 8 +- .../libp2p_stubs/crypto/pb/crypto_pb2.py | 59 +++++---- .../libp2p_stubs/crypto/pb/crypto_pb2.pyi | 43 ++++-- src/aleph/toolkit/libp2p_stubs/crypto/rsa.py | 2 +- src/aleph/toolkit/libp2p_stubs/peer/id.py | 8 +- src/aleph/toolkit/logging.py | 2 +- src/aleph/toolkit/range.py | 3 +- src/aleph/toolkit/shield.py | 1 + src/aleph/toolkit/split.py | 5 +- src/aleph/toolkit/timestamp.py | 2 +- src/aleph/types/message_processing_result.py | 12 +- src/aleph/types/message_status.py | 4 +- src/aleph/types/protocol.py | 1 + src/aleph/web/controllers/accounts.py | 7 +- src/aleph/web/controllers/aggregates.py | 11 +- .../web/controllers/app_state_getters.py | 4 +- src/aleph/web/controllers/channels.py | 2 +- src/aleph/web/controllers/ipfs.py | 8 +- src/aleph/web/controllers/messages.py | 9 +- src/aleph/web/controllers/metrics.py | 6 +- src/aleph/web/controllers/p2p.py | 6 +- src/aleph/web/controllers/posts.py | 7 +- src/aleph/web/controllers/prices.py | 10 +- src/aleph/web/controllers/routes.py | 2 +- src/aleph/web/controllers/storage.py | 55 ++++---- src/aleph/web/controllers/utils.py | 20 ++- src/aleph/web/controllers/version.py | 1 + tests/api/conftest.py | 8 +- tests/api/test_aggregates.py | 5 +- tests/api/test_balance.py | 7 +- tests/api/test_get_message.py | 12 +- tests/api/test_list_messages.py | 11 +- tests/api/test_new_metric.py | 6 +- tests/api/test_posts.py | 4 +- tests/api/test_storage.py | 17 ++- tests/api/utils/__init__.py | 2 +- tests/balances/test_balances.py | 7 +- tests/chains/test_aleph_indexer.py | 2 +- tests/chains/test_avalanche.py | 2 +- tests/chains/test_chain_data_service.py | 11 +- tests/chains/test_common.py | 11 +- tests/chains/test_confirmation.py | 28 ++-- tests/chains/test_cosmos.py | 3 +- tests/chains/test_nuls2.py | 5 +- tests/chains/test_solana.py | 4 +- tests/chains/test_substrate.py | 1 - tests/chains/test_tezos.py | 10 +- tests/conftest.py | 21 ++- tests/db/test_aggregates.py | 16 +-- tests/db/test_chains.py | 13 +- tests/db/test_cost.py | 22 ++-- tests/db/test_files.py | 8 +- tests/db/test_messages.py | 45 +++---- tests/db/test_peers.py | 2 +- tests/db/test_pending_messages_db.py | 6 +- tests/db/test_pending_txs.py | 5 +- tests/db/test_posts.py | 24 ++-- tests/db/test_programs_db.py | 14 +- tests/helpers/message_test_helpers.py | 4 +- tests/message_processing/conftest.py | 4 +- .../test_process_aggregates.py | 12 +- .../test_process_forgets.py | 20 ++- .../test_process_instances.py | 27 ++-- .../test_process_pending_txs.py | 3 +- .../test_process_programs.py | 36 +++-- .../message_processing/test_process_stores.py | 7 +- .../test_check_sender_authorization.py | 2 +- tests/services/test_cost_service.py | 123 ++++++++---------- tests/services/test_garbage_collector.py | 7 +- tests/services/test_utils.py | 4 +- tests/storage/test_get_content.py | 7 +- tests/storage/test_store_message.py | 4 +- tests/test_network.py | 1 - tests/toolkit/test_batch.py | 1 + tests/toolkit/test_range.py | 3 +- tests/toolkit/test_timer.py | 3 +- tests/web/controllers/test_metrics.py | 68 +++++----- tests/web/controllers/test_programs.py | 2 +- 170 files changed, 888 insertions(+), 929 deletions(-) diff --git a/deployment/migrations/env.py b/deployment/migrations/env.py index c90c7b72f..aed2c989a 100644 --- a/deployment/migrations/env.py +++ b/deployment/migrations/env.py @@ -17,7 +17,7 @@ fileConfig(config.config_file_name) # Auto-generate migrations -from aleph.db.models import Base +from aleph.db.models import Base # noqa target_metadata = Base.metadata diff --git a/deployment/migrations/versions/0007_0bfde82697c8_balance_views.py b/deployment/migrations/versions/0007_0bfde82697c8_balance_views.py index a3ba758fb..ab7467ecb 100644 --- a/deployment/migrations/versions/0007_0bfde82697c8_balance_views.py +++ b/deployment/migrations/versions/0007_0bfde82697c8_balance_views.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0008_eef5f95853bf_trusted_messages.py b/deployment/migrations/versions/0008_eef5f95853bf_trusted_messages.py index d36c3bc44..95a61e50e 100644 --- a/deployment/migrations/versions/0008_eef5f95853bf_trusted_messages.py +++ b/deployment/migrations/versions/0008_eef5f95853bf_trusted_messages.py @@ -7,7 +7,6 @@ """ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "eef5f95853bf" diff --git a/deployment/migrations/versions/0009_8edf69c47884_file_pin_owner_index.py b/deployment/migrations/versions/0009_8edf69c47884_file_pin_owner_index.py index 41adec817..df5cb6552 100644 --- a/deployment/migrations/versions/0009_8edf69c47884_file_pin_owner_index.py +++ b/deployment/migrations/versions/0009_8edf69c47884_file_pin_owner_index.py @@ -7,7 +7,6 @@ """ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "8edf69c47884" diff --git a/deployment/migrations/versions/0010_8a5eaab15d40_address_stats_view.py b/deployment/migrations/versions/0010_8a5eaab15d40_address_stats_view.py index dca234715..c8187b787 100644 --- a/deployment/migrations/versions/0010_8a5eaab15d40_address_stats_view.py +++ b/deployment/migrations/versions/0010_8a5eaab15d40_address_stats_view.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0013_7ab62bd0a3b1_messages_post_index.py b/deployment/migrations/versions/0013_7ab62bd0a3b1_messages_post_index.py index 597307208..af9fad132 100644 --- a/deployment/migrations/versions/0013_7ab62bd0a3b1_messages_post_index.py +++ b/deployment/migrations/versions/0013_7ab62bd0a3b1_messages_post_index.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0014_daa92b500049_message_content_in_file_pins.py b/deployment/migrations/versions/0014_daa92b500049_message_content_in_file_pins.py index 098c4e80a..22b90d760 100644 --- a/deployment/migrations/versions/0014_daa92b500049_message_content_in_file_pins.py +++ b/deployment/migrations/versions/0014_daa92b500049_message_content_in_file_pins.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0016_77e68941d36c_fix_ipfs_files_size.py b/deployment/migrations/versions/0016_77e68941d36c_fix_ipfs_files_size.py index c520052be..0baf3f10c 100644 --- a/deployment/migrations/versions/0016_77e68941d36c_fix_ipfs_files_size.py +++ b/deployment/migrations/versions/0016_77e68941d36c_fix_ipfs_files_size.py @@ -8,7 +8,6 @@ import asyncio import logging from threading import Thread -from time import sleep import aioipfs from alembic import op diff --git a/deployment/migrations/versions/0018_7bcb8e5fe186_fix_vm_cost_view.py b/deployment/migrations/versions/0018_7bcb8e5fe186_fix_vm_cost_view.py index ecfaea64d..fa1861a36 100644 --- a/deployment/migrations/versions/0018_7bcb8e5fe186_fix_vm_cost_view.py +++ b/deployment/migrations/versions/0018_7bcb8e5fe186_fix_vm_cost_view.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0021_08602db6c78f_ccn_metric_view.py b/deployment/migrations/versions/0021_08602db6c78f_ccn_metric_view.py index 20283a1ae..fd1bf48db 100644 --- a/deployment/migrations/versions/0021_08602db6c78f_ccn_metric_view.py +++ b/deployment/migrations/versions/0021_08602db6c78f_ccn_metric_view.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa # revision identifiers, used by Alembic. diff --git a/deployment/migrations/versions/0022_e682fc8f9506_fix_vm_costs_view.py b/deployment/migrations/versions/0022_e682fc8f9506_fix_vm_costs_view.py index e86b0bc9c..4450a8cbc 100644 --- a/deployment/migrations/versions/0022_e682fc8f9506_fix_vm_costs_view.py +++ b/deployment/migrations/versions/0022_e682fc8f9506_fix_vm_costs_view.py @@ -6,7 +6,6 @@ """ from alembic import op -import sqlalchemy as sa revision = 'e682fc8f9506' diff --git a/pyproject.toml b/pyproject.toml index 0bf4b7324..340c850c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,7 +169,7 @@ dependencies = [ [tool.hatch.envs.linting.scripts] typing = "mypy --config-file=pyproject.toml {args:} ./src/ ./tests/" style = [ - "ruff check {args:.} ./src/ ./tests/", + "ruff check {args:.}", "black --check --diff {args:} ./src/ ./tests/", "isort --check-only --profile black {args:} ./src/ ./tests/", ] diff --git a/src/aleph/__init__.py b/src/aleph/__init__.py index 473ee9f84..4c8f1d779 100644 --- a/src/aleph/__init__.py +++ b/src/aleph/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import subprocess -from pkg_resources import get_distribution, DistributionNotFound +from pkg_resources import DistributionNotFound, get_distribution def _get_git_version() -> str: diff --git a/src/aleph/api_entrypoint.py b/src/aleph/api_entrypoint.py index ed97ec57a..106e98b83 100644 --- a/src/aleph/api_entrypoint.py +++ b/src/aleph/api_entrypoint.py @@ -10,7 +10,6 @@ from aleph.db.connection import make_engine, make_session_factory from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService -from aleph.services.ipfs.common import make_ipfs_client from aleph.services.p2p import init_p2p_client from aleph.services.storage.fileystem_engine import FileSystemStorageEngine from aleph.storage import StorageService @@ -18,27 +17,27 @@ from aleph.web import create_aiohttp_app from aleph.web.controllers.app_state_getters import ( APP_STATE_CONFIG, + APP_STATE_MQ_CHANNEL, APP_STATE_MQ_CONN, + APP_STATE_MQ_WS_CHANNEL, APP_STATE_NODE_CACHE, APP_STATE_P2P_CLIENT, APP_STATE_SESSION_FACTORY, - APP_STATE_STORAGE_SERVICE, - APP_STATE_MQ_CHANNEL, - APP_STATE_MQ_WS_CHANNEL, APP_STATE_SIGNATURE_VERIFIER, + APP_STATE_STORAGE_SERVICE, ) async def configure_aiohttp_app( config: Config, ) -> web.Application: - with sentry_sdk.start_transaction(name=f"init-api-server"): - p2p_client = await init_p2p_client(config, service_name=f"api-server-aiohttp") + with sentry_sdk.start_transaction(name="init-api-server"): + p2p_client = await init_p2p_client(config, service_name="api-server-aiohttp") engine = make_engine( config, echo=config.logging.level.value == logging.DEBUG, - application_name=f"aleph-api", + application_name="aleph-api", ) session_factory = make_session_factory(engine) diff --git a/src/aleph/chains/abc.py b/src/aleph/chains/abc.py index 949d25536..01e08a28e 100644 --- a/src/aleph/chains/abc.py +++ b/src/aleph/chains/abc.py @@ -3,22 +3,18 @@ from configmanager import Config from aleph.schemas.pending_messages import BasePendingMessage -from aleph.types.chain_sync import ChainEventType class Verifier(abc.ABC): @abc.abstractmethod - async def verify_signature(self, message: BasePendingMessage) -> bool: - ... + async def verify_signature(self, message: BasePendingMessage) -> bool: ... class ChainReader(abc.ABC): @abc.abstractmethod - async def fetcher(self, config: Config): - ... + async def fetcher(self, config: Config): ... class ChainWriter(ChainReader): @abc.abstractmethod - async def packer(self, config: Config): - ... + async def packer(self, config: Config): ... diff --git a/src/aleph/chains/avalanche.py b/src/aleph/chains/avalanche.py index a872cef1e..12e2ca217 100644 --- a/src/aleph/chains/avalanche.py +++ b/src/aleph/chains/avalanche.py @@ -8,6 +8,7 @@ from aleph.chains.common import get_verification_buffer from aleph.schemas.pending_messages import BasePendingMessage + from .abc import Verifier LOGGER = logging.getLogger("chains.avalanche") @@ -86,7 +87,7 @@ async def verify_signature(self, message: BasePendingMessage) -> bool: result = address == message.sender - except Exception as e: + except Exception: LOGGER.exception("Error processing signature for %s" % message.sender) result = False diff --git a/src/aleph/chains/chain_data_service.py b/src/aleph/chains/chain_data_service.py index 9f3562f78..b510c8a80 100644 --- a/src/aleph/chains/chain_data_service.py +++ b/src/aleph/chains/chain_data_service.py @@ -1,29 +1,28 @@ import asyncio -from io import StringIO -from typing import Dict, Optional, List, Any, Mapping, Set, cast, Type, Union, Self +from typing import Any, Dict, List, Mapping, Optional, Self, Set, Type, Union, cast import aio_pika.abc -from aleph_message.models import StoreContent, ItemType, Chain, MessageType +from aleph_message.models import Chain, ItemType, MessageType, StoreContent from configmanager import Config from pydantic import ValidationError from aleph.chains.common import LOGGER from aleph.config import get_config from aleph.db.accessors.chains import upsert_chain_tx -from aleph.db.accessors.files import upsert_tx_file_pin, upsert_file +from aleph.db.accessors.files import upsert_file, upsert_tx_file_pin from aleph.db.accessors.pending_txs import upsert_pending_tx from aleph.db.models import ChainTxDb, MessageDb from aleph.exceptions import ( - InvalidContent, AlephStorageException, ContentCurrentlyUnavailable, + InvalidContent, ) -from aleph.schemas.chains.indexer_response import MessageEvent, GenericMessageEvent +from aleph.schemas.chains.indexer_response import GenericMessageEvent, MessageEvent from aleph.schemas.chains.sync_events import ( OffChainSyncEventPayload, - OnChainSyncEventPayload, OnChainContent, OnChainMessage, + OnChainSyncEventPayload, ) from aleph.schemas.chains.tezos_indexer_response import ( MessageEventPayload as TezosMessageEventPayload, @@ -31,7 +30,7 @@ from aleph.storage import StorageService from aleph.toolkit.timestamp import utc_now from aleph.types.chain_sync import ChainSyncProtocol -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.files import FileType from aleph.utils import get_sha256 diff --git a/src/aleph/chains/common.py b/src/aleph/chains/common.py index c72d3b276..f4a13f44e 100644 --- a/src/aleph/chains/common.py +++ b/src/aleph/chains/common.py @@ -1,8 +1,6 @@ import logging -from aleph.schemas.pending_messages import ( - BasePendingMessage, -) +from aleph.schemas.pending_messages import BasePendingMessage LOGGER = logging.getLogger("chains.common") diff --git a/src/aleph/chains/connector.py b/src/aleph/chains/connector.py index 5018dd5bf..e612f079f 100644 --- a/src/aleph/chains/connector.py +++ b/src/aleph/chains/connector.py @@ -6,6 +6,7 @@ from configmanager import Config from aleph.types.db_session import DbSessionFactory + from .abc import ChainReader, ChainWriter from .bsc import BscConnector from .chain_data_service import ChainDataService, PendingTxPublisher diff --git a/src/aleph/chains/cosmos.py b/src/aleph/chains/cosmos.py index 9be11b1b3..03fdcabae 100644 --- a/src/aleph/chains/cosmos.py +++ b/src/aleph/chains/cosmos.py @@ -10,6 +10,7 @@ from aleph.chains.common import get_verification_buffer from aleph.schemas.pending_messages import BasePendingMessage + from .abc import Verifier LOGGER = logging.getLogger("chains.cosmos") diff --git a/src/aleph/chains/ethereum.py b/src/aleph/chains/ethereum.py index ea699e21c..b3e0c8230 100644 --- a/src/aleph/chains/ethereum.py +++ b/src/aleph/chains/ethereum.py @@ -29,6 +29,7 @@ from aleph.types.chain_sync import ChainEventType from aleph.types.db_session import DbSessionFactory from aleph.utils import run_in_executor + from .abc import ChainWriter, Verifier from .chain_data_service import ChainDataService, PendingTxPublisher from .indexer_reader import AlephIndexerReader @@ -94,7 +95,7 @@ async def verify_signature(self, message: BasePendingMessage) -> bool: ) return False - except Exception as e: + except Exception: LOGGER.exception("Error processing signature for %s" % message.sender) verified = False diff --git a/src/aleph/chains/indexer_reader.py b/src/aleph/chains/indexer_reader.py index a11419873..808aab374 100644 --- a/src/aleph/chains/indexer_reader.py +++ b/src/aleph/chains/indexer_reader.py @@ -4,16 +4,16 @@ import logging from dataclasses import dataclass from typing import ( - Mapping, + Any, + Dict, + Iterable, List, + Mapping, Optional, Tuple, - TypeVar, Type, + TypeVar, Union, - Dict, - Any, - Iterable, ) import aiohttp @@ -23,22 +23,22 @@ import aleph.toolkit.json as aleph_json from aleph.chains.chain_data_service import PendingTxPublisher from aleph.db.accessors.chains import ( - get_missing_indexer_datetime_multirange, add_indexer_range, + get_missing_indexer_datetime_multirange, ) from aleph.db.models import ChainTxDb from aleph.schemas.chains.indexer_response import ( EntityType, - IndexerBlockchain, IndexerAccountStateResponse, + IndexerBlockchain, IndexerEventResponse, MessageEvent, SyncEvent, ) -from aleph.toolkit.range import Range, MultiRange +from aleph.toolkit.range import MultiRange, Range from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.chain_sync import ChainEventType, ChainSyncProtocol -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/chains/nuls.py b/src/aleph/chains/nuls.py index 8cb0744f7..84c11d442 100644 --- a/src/aleph/chains/nuls.py +++ b/src/aleph/chains/nuls.py @@ -1,18 +1,18 @@ import logging import struct +from aleph.chains.common import get_verification_buffer +from aleph.schemas.pending_messages import BasePendingMessage +from aleph.utils import run_in_executor + +from .abc import Verifier from .nuls_aleph_sdk import ( NulsSignature, + address_from_hash, hash_from_address, public_key_to_hash, - address_from_hash, ) -from aleph.chains.common import get_verification_buffer -from aleph.schemas.pending_messages import BasePendingMessage -from aleph.utils import run_in_executor -from .abc import Verifier - LOGGER = logging.getLogger("chains.nuls") CHAIN_NAME = "NULS" diff --git a/src/aleph/chains/nuls2.py b/src/aleph/chains/nuls2.py index 39f9b8ec4..a01871304 100644 --- a/src/aleph/chains/nuls2.py +++ b/src/aleph/chains/nuls2.py @@ -14,10 +14,10 @@ from configmanager import Config from nuls2.api.server import get_server from nuls2.model.data import ( + CHEAP_UNIT_FEE, + get_address, hash_from_address, recover_message_address, - get_address, - CHEAP_UNIT_FEE, ) from nuls2.model.transaction import Transaction @@ -26,15 +26,16 @@ from aleph.db.accessors.messages import get_unconfirmed_messages from aleph.db.accessors.pending_messages import count_pending_messages from aleph.db.accessors.pending_txs import count_pending_txs +from aleph.schemas.chains.tx_context import TxContext from aleph.schemas.pending_messages import BasePendingMessage from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSessionFactory from aleph.utils import run_in_executor -from .chain_data_service import ChainDataService, PendingTxPublisher -from .abc import Verifier, ChainWriter -from aleph.schemas.chains.tx_context import TxContext + from ..db.models import ChainTxDb from ..types.chain_sync import ChainEventType +from .abc import ChainWriter, Verifier +from .chain_data_service import ChainDataService, PendingTxPublisher LOGGER = logging.getLogger("chains.nuls2") CHAIN_NAME = "NULS2" diff --git a/src/aleph/chains/nuls_aleph_sdk.py b/src/aleph/chains/nuls_aleph_sdk.py index 815d43ed0..7c17257c4 100644 --- a/src/aleph/chains/nuls_aleph_sdk.py +++ b/src/aleph/chains/nuls_aleph_sdk.py @@ -1,6 +1,7 @@ """ Code imported from aleph-client to avoid a direct reference to the SDK. """ + import hashlib import logging import struct @@ -82,7 +83,6 @@ def sizeOf(cls, value): # // * @return the minimal encoded bytes of the value # // */ def encode(self): - ob = bytes() size = self.sizeOf(self.value) if size == 1: diff --git a/src/aleph/chains/signature_verifier.py b/src/aleph/chains/signature_verifier.py index 6cbba9887..02da09d79 100644 --- a/src/aleph/chains/signature_verifier.py +++ b/src/aleph/chains/signature_verifier.py @@ -2,8 +2,8 @@ from aleph_message.models import Chain -from aleph.chains.avalanche import AvalancheConnector from aleph.chains.abc import Verifier +from aleph.chains.avalanche import AvalancheConnector from aleph.chains.ethereum import EthereumVerifier from aleph.chains.nuls import NulsConnector from aleph.chains.nuls2 import Nuls2Verifier diff --git a/src/aleph/chains/solana.py b/src/aleph/chains/solana.py index 2fad61e6f..95938f64d 100644 --- a/src/aleph/chains/solana.py +++ b/src/aleph/chains/solana.py @@ -7,6 +7,7 @@ from aleph.chains.common import get_verification_buffer from aleph.schemas.pending_messages import BasePendingMessage + from .abc import Verifier LOGGER = logging.getLogger("chains.solana") diff --git a/src/aleph/chains/substrate.py b/src/aleph/chains/substrate.py index 76288844d..839730c56 100644 --- a/src/aleph/chains/substrate.py +++ b/src/aleph/chains/substrate.py @@ -5,6 +5,7 @@ from aleph.chains.common import get_verification_buffer from aleph.schemas.pending_messages import BasePendingMessage + from .abc import Verifier LOGGER = logging.getLogger("chains.substrate") diff --git a/src/aleph/chains/tezos.py b/src/aleph/chains/tezos.py index 279c1cb2c..a399384a0 100644 --- a/src/aleph/chains/tezos.py +++ b/src/aleph/chains/tezos.py @@ -11,20 +11,20 @@ from nacl.exceptions import BadSignatureError import aleph.toolkit.json as aleph_json -from aleph.chains.abc import Verifier, ChainReader +from aleph.chains.abc import ChainReader, Verifier from aleph.chains.chain_data_service import PendingTxPublisher from aleph.chains.common import get_verification_buffer from aleph.db.accessors.chains import get_last_height, upsert_chain_sync_status -from aleph.db.models import PendingMessageDb, ChainTxDb +from aleph.db.models import ChainTxDb, PendingMessageDb from aleph.schemas.chains.tezos_indexer_response import ( - IndexerResponse, IndexerMessageEvent, + IndexerResponse, SyncStatus, ) from aleph.schemas.pending_messages import BasePendingMessage from aleph.toolkit.timestamp import utc_now -from aleph.types.chain_sync import ChainSyncProtocol, ChainEventType -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.chain_sync import ChainEventType, ChainSyncProtocol +from aleph.types.db_session import DbSession, DbSessionFactory LOGGER = logging.getLogger(__name__) @@ -231,7 +231,7 @@ async def verify_signature(self, message: BasePendingMessage) -> bool: ) verification_buffer = get_tezos_verification_buffer( - message, signature_type, dapp_url # type: ignore + message, signature_type, dapp_url ) # Check the signature diff --git a/src/aleph/commands.py b/src/aleph/commands.py index 95794d479..31c8878e5 100644 --- a/src/aleph/commands.py +++ b/src/aleph/commands.py @@ -26,7 +26,7 @@ from aleph.chains.chain_data_service import ChainDataService, PendingTxPublisher from aleph.chains.connector import ChainConnector from aleph.cli.args import parse_args -from aleph.db.connection import make_engine, make_session_factory, make_db_url +from aleph.db.connection import make_db_url, make_engine, make_session_factory from aleph.exceptions import InvalidConfigException, KeyNotFoundException from aleph.jobs import start_jobs from aleph.network import listener_tasks @@ -37,8 +37,8 @@ from aleph.services.keys import generate_keypair, save_keys from aleph.services.storage.fileystem_engine import FileSystemStorageEngine from aleph.services.storage.garbage_collector import ( - garbage_collector_task, GarbageCollector, + garbage_collector_task, ) from aleph.storage import StorageService from aleph.toolkit.logging import setup_logging diff --git a/src/aleph/db/accessors/aggregates.py b/src/aleph/db/accessors/aggregates.py index 1c64fe473..6fa7d7cc1 100644 --- a/src/aleph/db/accessors/aggregates.py +++ b/src/aleph/db/accessors/aggregates.py @@ -1,24 +1,19 @@ import datetime as dt from typing import ( - Optional, - Iterable, Any, Dict, - Tuple, - Sequence, - overload, + Iterable, Literal, + Optional, + Sequence, + Tuple, Union, + overload, ) -from sqlalchemy import ( - select, - delete, - update, - func, - literal_column, -) -from sqlalchemy.dialects.postgresql import insert, aggregate_order_by -from sqlalchemy.orm import selectinload, defer + +from sqlalchemy import delete, func, literal_column, select, update +from sqlalchemy.dialects.postgresql import aggregate_order_by, insert +from sqlalchemy.orm import defer, selectinload from aleph.db.models import AggregateDb, AggregateElementDb from aleph.types.db_session import DbSession @@ -41,8 +36,7 @@ def get_aggregates_by_owner( owner: str, with_info: Literal[False], keys: Optional[Sequence[str]] = None, -) -> AggregateContent: - ... +) -> AggregateContent: ... @overload @@ -51,15 +45,13 @@ def get_aggregates_by_owner( owner: str, with_info: Literal[True], keys: Optional[Sequence[str]] = None, -) -> AggregateContentWithInfo: - ... +) -> AggregateContentWithInfo: ... @overload def get_aggregates_by_owner( session, owner: str, with_info: bool, keys: Optional[Sequence[str]] = None -) -> Union[AggregateContent, AggregateContentWithInfo]: - ... +) -> Union[AggregateContent, AggregateContentWithInfo]: ... def get_aggregates_by_owner(session, owner, with_info, keys=None): diff --git a/src/aleph/db/accessors/balances.py b/src/aleph/db/accessors/balances.py index f80ee2c2f..86455e248 100644 --- a/src/aleph/db/accessors/balances.py +++ b/src/aleph/db/accessors/balances.py @@ -1,9 +1,9 @@ from decimal import Decimal from io import StringIO -from typing import Optional, Mapping +from typing import Mapping, Optional from aleph_message.models import Chain -from sqlalchemy import select, func +from sqlalchemy import func, select from aleph.db.models import AlephBalanceDb from aleph.types.db_session import DbSession diff --git a/src/aleph/db/accessors/chains.py b/src/aleph/db/accessors/chains.py index 54a913965..9da9bbbfa 100644 --- a/src/aleph/db/accessors/chains.py +++ b/src/aleph/db/accessors/chains.py @@ -1,16 +1,17 @@ import datetime as dt from dataclasses import dataclass -from typing import Optional, Tuple, Iterable +from typing import Iterable, Optional from aleph_message.models import Chain -from sqlalchemy import select, delete +from sqlalchemy import delete, select from sqlalchemy.dialects.postgresql import insert from aleph.toolkit.range import MultiRange, Range from aleph.toolkit.timestamp import utc_now from aleph.types.chain_sync import ChainEventType from aleph.types.db_session import DbSession -from ..models.chains import ChainSyncStatusDb, IndexerSyncStatusDb, ChainTxDb + +from ..models.chains import ChainSyncStatusDb, ChainTxDb, IndexerSyncStatusDb def get_last_height( diff --git a/src/aleph/db/accessors/cost.py b/src/aleph/db/accessors/cost.py index 55e4a7eec..b9a255216 100644 --- a/src/aleph/db/accessors/cost.py +++ b/src/aleph/db/accessors/cost.py @@ -1,6 +1,7 @@ from decimal import Decimal -from typing import Optional -from sqlalchemy import select, func, text + +from sqlalchemy import func, select, text + from aleph.types.db_session import DbSession diff --git a/src/aleph/db/accessors/files.py b/src/aleph/db/accessors/files.py index cea15418e..7b0f036f5 100644 --- a/src/aleph/db/accessors/files.py +++ b/src/aleph/db/accessors/files.py @@ -1,5 +1,5 @@ import datetime as dt -from typing import Optional, Iterable, Collection, Tuple +from typing import Collection, Iterable, Optional, Tuple from sqlalchemy import delete, func, select from sqlalchemy.dialects.postgresql import insert @@ -8,15 +8,16 @@ from aleph.types.db_session import DbSession from aleph.types.files import FileTag, FileType from aleph.types.sort_order import SortOrder + from ..models.files import ( + ContentFilePinDb, FilePinDb, + FilePinType, FileTagDb, + GracePeriodFilePinDb, + MessageFilePinDb, StoredFileDb, TxFilePinDb, - MessageFilePinDb, - FilePinType, - ContentFilePinDb, - GracePeriodFilePinDb, ) diff --git a/src/aleph/db/accessors/messages.py b/src/aleph/db/accessors/messages.py index 198ecb08a..f03be4535 100644 --- a/src/aleph/db/accessors/messages.py +++ b/src/aleph/db/accessors/messages.py @@ -1,11 +1,11 @@ import datetime as dt import traceback -from typing import Optional, Sequence, Union, Iterable, Any, Mapping, overload, Tuple +from typing import Any, Iterable, Mapping, Optional, Sequence, Tuple, Union, overload -from aleph_message.models import ItemHash, Chain, MessageType -from sqlalchemy import func, select, update, text, delete, nullsfirst, nullslast -from sqlalchemy.dialects.postgresql import insert, array -from sqlalchemy.orm import selectinload, load_only +from aleph_message.models import Chain, ItemHash, MessageType +from sqlalchemy import delete, func, nullsfirst, nullslast, select, text, update +from sqlalchemy.dialects.postgresql import array, insert +from sqlalchemy.orm import load_only, selectinload from sqlalchemy.sql import Insert, Select from sqlalchemy.sql.elements import literal @@ -13,24 +13,27 @@ from aleph.types.channel import Channel from aleph.types.db_session import DbSession from aleph.types.message_status import ( - MessageStatus, - MessageProcessingException, ErrorCode, + MessageProcessingException, + MessageStatus, ) -from aleph.types.sort_order import SortOrder, SortBy -from .pending_messages import delete_pending_message +from aleph.types.sort_order import SortBy, SortOrder + from ..models.chains import ChainTxDb from ..models.messages import ( + ForgottenMessageDb, MessageDb, MessageStatusDb, - ForgottenMessageDb, RejectedMessageDb, message_confirmations, ) from ..models.pending_messages import PendingMessageDb +from .pending_messages import delete_pending_message -def get_message_by_item_hash(session: DbSession, item_hash: ItemHash) -> Optional[MessageDb]: +def get_message_by_item_hash( + session: DbSession, item_hash: ItemHash +) -> Optional[MessageDb]: select_stmt = ( select(MessageDb) .where(MessageDb.item_hash == item_hash) @@ -133,7 +136,9 @@ def make_matching_messages_query( ) if start_block: select_stmt = select_stmt.where( - select_earliest_confirmation.c.height.is_(None) | select_earliest_confirmation.c.height >= start_block + select_earliest_confirmation.c.height.is_(None) + | select_earliest_confirmation.c.height + >= start_block ) if end_block: select_stmt = select_stmt.where( @@ -142,12 +147,16 @@ def make_matching_messages_query( if sort_by == SortBy.TX_TIME: order_by_columns = ( ( - nullsfirst(select_earliest_confirmation.c.earliest_confirmation.desc()), + nullsfirst( + select_earliest_confirmation.c.earliest_confirmation.desc() + ), MessageDb.time.desc(), ) if sort_order == SortOrder.DESCENDING else ( - nullslast(select_earliest_confirmation.c.earliest_confirmation.asc()), + nullslast( + select_earliest_confirmation.c.earliest_confirmation.asc() + ), MessageDb.time.asc(), ) ) @@ -289,7 +298,9 @@ def make_confirmation_upsert_query(item_hash: str, tx_hash: str) -> Insert: ) -def get_message_status(session: DbSession, item_hash: ItemHash) -> Optional[MessageStatusDb]: +def get_message_status( + session: DbSession, item_hash: ItemHash +) -> Optional[MessageStatusDb]: return ( session.execute( select(MessageStatusDb).where(MessageStatusDb.item_hash == str(item_hash)) @@ -488,9 +499,8 @@ def reject_new_pending_message( session: DbSession, pending_message: Mapping[str, Any], exception: BaseException, - tx_hash: Optional[str], -) -> None: - ... + tx_hash: Optional[str], +) -> None: ... @overload @@ -499,8 +509,7 @@ def reject_new_pending_message( pending_message: PendingMessageDb, exception: BaseException, tx_hash: Optional[str], -) -> None: - ... +) -> None: ... def reject_new_pending_message( diff --git a/src/aleph/db/accessors/peers.py b/src/aleph/db/accessors/peers.py index f28ea2d9a..5d30533f9 100644 --- a/src/aleph/db/accessors/peers.py +++ b/src/aleph/db/accessors/peers.py @@ -6,10 +6,13 @@ from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSession + from ..models.peers import PeerDb, PeerType -def get_all_addresses_by_peer_type(session: DbSession, peer_type: PeerType) -> Sequence[str]: +def get_all_addresses_by_peer_type( + session: DbSession, peer_type: PeerType +) -> Sequence[str]: select_peers_stmt = select(PeerDb.address).where(PeerDb.peer_type == peer_type) addresses = session.execute(select_peers_stmt) diff --git a/src/aleph/db/accessors/pending_messages.py b/src/aleph/db/accessors/pending_messages.py index dacd0601e..50b1c0a48 100644 --- a/src/aleph/db/accessors/pending_messages.py +++ b/src/aleph/db/accessors/pending_messages.py @@ -1,12 +1,12 @@ import datetime as dt -from typing import Optional, Iterable, Any, Dict, Sequence, Collection +from typing import Any, Collection, Dict, Iterable, Optional, Sequence from aleph_message.models import Chain -from sqlalchemy import select, func, update, delete +from sqlalchemy import delete, func, select, update from sqlalchemy.orm import selectinload from sqlalchemy.sql import Update -from aleph.db.models import PendingMessageDb, ChainTxDb +from aleph.db.models import ChainTxDb, PendingMessageDb from aleph.types.db_session import DbSession @@ -76,7 +76,9 @@ def get_pending_messages( return session.execute(select_stmt).scalars() -def get_pending_message(session: DbSession, pending_message_id: int) -> Optional[PendingMessageDb]: +def get_pending_message( + session: DbSession, pending_message_id: int +) -> Optional[PendingMessageDb]: select_stmt = select(PendingMessageDb).where( PendingMessageDb.id == pending_message_id ) diff --git a/src/aleph/db/accessors/pending_txs.py b/src/aleph/db/accessors/pending_txs.py index ada9802fc..5d1ce9f83 100644 --- a/src/aleph/db/accessors/pending_txs.py +++ b/src/aleph/db/accessors/pending_txs.py @@ -1,11 +1,11 @@ -from typing import Optional, Iterable +from typing import Iterable, Optional from aleph_message.models import Chain -from sqlalchemy import select, func, delete +from sqlalchemy import delete, func, select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import selectinload -from aleph.db.models import PendingTxDb, ChainTxDb +from aleph.db.models import ChainTxDb, PendingTxDb from aleph.types.db_session import DbSession diff --git a/src/aleph/db/accessors/posts.py b/src/aleph/db/accessors/posts.py index eb30e3114..655339add 100644 --- a/src/aleph/db/accessors/posts.py +++ b/src/aleph/db/accessors/posts.py @@ -1,43 +1,40 @@ import datetime as dt from typing import ( + Any, + Dict, + Iterable, + List, Optional, Protocol, - Dict, - Any, Sequence, + Tuple, Union, - List, cast, - Iterable, - Tuple, ) -from aleph_message.models import ItemHash, Chain, ItemType +from aleph_message.models import Chain, ItemHash, ItemType +from sqlalchemy import TIMESTAMP, Float, String, case +from sqlalchemy import cast as sqla_cast from sqlalchemy import ( + delete, + extract, func, - select, literal_column, - TIMESTAMP, - String, - delete, - update, nullsfirst, nullslast, - extract, - cast as sqla_cast, - Float, - case, + select, + update, ) from sqlalchemy.dialects.postgresql import JSONB, array from sqlalchemy.orm import aliased from sqlalchemy.sql import Select -from aleph.db.models import message_confirmations, ChainTxDb, MessageDb +from aleph.db.models import ChainTxDb, MessageDb, message_confirmations from aleph.db.models.posts import PostDb from aleph.toolkit.timestamp import coerce_to_datetime from aleph.types.channel import Channel from aleph.types.db_session import DbSession -from aleph.types.sort_order import SortOrder, SortBy +from aleph.types.sort_order import SortBy, SortOrder class MergedPost(Protocol): diff --git a/src/aleph/db/accessors/vms.py b/src/aleph/db/accessors/vms.py index 90f7fa992..7a9a7044f 100644 --- a/src/aleph/db/accessors/vms.py +++ b/src/aleph/db/accessors/vms.py @@ -1,15 +1,10 @@ import datetime as dt -from typing import Optional, Iterable -from decimal import Decimal -from sqlalchemy import delete, func, select, text +from typing import Iterable, Optional + +from sqlalchemy import delete, func, select from sqlalchemy.dialects.postgresql import insert -from aleph.db.models.vms import ( - VmBaseDb, - VmVersionDb, - ProgramDb, - VmInstanceDb, -) +from aleph.db.models.vms import ProgramDb, VmBaseDb, VmInstanceDb, VmVersionDb from aleph.types.db_session import DbSession from aleph.types.vms import VmVersion diff --git a/src/aleph/db/connection.py b/src/aleph/db/connection.py index 953911e76..85f2cad6e 100644 --- a/src/aleph/db/connection.py +++ b/src/aleph/db/connection.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import sessionmaker from aleph.config import get_config -from aleph.types.db_session import DbSessionFactory, AsyncDbSessionFactory +from aleph.types.db_session import AsyncDbSessionFactory, DbSessionFactory def make_db_url( @@ -38,9 +38,7 @@ def make_db_url( if host is not None: connection_string += f"{host}:{port}" - connection_string += ( - f"/{database}" - ) + connection_string += f"/{database}" if application_name: connection_string += f"?application_name={application_name}" diff --git a/src/aleph/db/models/__init__.py b/src/aleph/db/models/__init__.py index 3ecbd37ae..e761f87c1 100644 --- a/src/aleph/db/models/__init__.py +++ b/src/aleph/db/models/__init__.py @@ -1,12 +1,11 @@ -from .base import Base - -from .aggregates import * -from .balances import * -from .chains import * -from .files import * -from .messages import * -from .peers import * -from .pending_messages import * -from .pending_txs import * -from .posts import * -from .vms import * +from .aggregates import * # noqa +from .balances import * # noqa +from .base import Base # noqa +from .chains import * # noqa +from .files import * # noqa +from .messages import * # noqa +from .peers import * # noqa +from .pending_messages import * # noqa +from .pending_txs import * # noqa +from .posts import * # noqa +from .vms import * # noqa diff --git a/src/aleph/db/models/aggregates.py b/src/aleph/db/models/aggregates.py index 0851a8eb5..53e519974 100644 --- a/src/aleph/db/models/aggregates.py +++ b/src/aleph/db/models/aggregates.py @@ -1,7 +1,7 @@ import datetime as dt from typing import Any -from sqlalchemy import Boolean, Column, ForeignKey, Index, String, TIMESTAMP +from sqlalchemy import TIMESTAMP, Boolean, Column, ForeignKey, Index, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship diff --git a/src/aleph/db/models/balances.py b/src/aleph/db/models/balances.py index 8c544174c..feff59d89 100644 --- a/src/aleph/db/models/balances.py +++ b/src/aleph/db/models/balances.py @@ -2,7 +2,7 @@ from typing import Optional from aleph_message.models import Chain -from sqlalchemy import Column, DECIMAL, String, Integer, UniqueConstraint, BigInteger +from sqlalchemy import DECIMAL, BigInteger, Column, Integer, String, UniqueConstraint from sqlalchemy_utils.types.choice import ChoiceType from .base import Base diff --git a/src/aleph/db/models/base.py b/src/aleph/db/models/base.py index 24b17507c..7dfe16781 100644 --- a/src/aleph/db/models/base.py +++ b/src/aleph/db/models/base.py @@ -1,6 +1,6 @@ -from typing import Dict, Any, Optional, Set, List, Iterable +from typing import Any, Dict, Iterable, Optional, Set -from sqlalchemy import Table, exists, text, Column, func, select +from sqlalchemy import Column, Table, exists, func, select, text from sqlalchemy.orm import declarative_base from aleph.types.db_session import DbSession diff --git a/src/aleph/db/models/chains.py b/src/aleph/db/models/chains.py index 3b8a1064b..e9bb8db6a 100644 --- a/src/aleph/db/models/chains.py +++ b/src/aleph/db/models/chains.py @@ -5,18 +5,19 @@ """ import datetime as dt -from typing import Dict, Any, Union, Mapping +from typing import Any, Dict, Mapping, Union from aleph_message.models import Chain -from sqlalchemy import Column, Integer, String, TIMESTAMP, Boolean +from sqlalchemy import TIMESTAMP, Boolean, Column, Integer, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy_utils.types.choice import ChoiceType -from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.chain_sync import ChainSyncProtocol, ChainEventType -from .base import Base from aleph.schemas.chains.tx_context import TxContext +from aleph.toolkit.timestamp import timestamp_to_datetime +from aleph.types.chain_sync import ChainEventType, ChainSyncProtocol + from ...toolkit.range import Range +from .base import Base class ChainSyncStatusDb(Base): diff --git a/src/aleph/db/models/files.py b/src/aleph/db/models/files.py index 0046b8d8e..8dac0178e 100644 --- a/src/aleph/db/models/files.py +++ b/src/aleph/db/models/files.py @@ -1,15 +1,22 @@ +import datetime as dt from enum import Enum -from typing import Optional, List, Any, Dict - -from sqlalchemy import BigInteger, Column, String, ForeignKey, TIMESTAMP, Index, UniqueConstraint +from typing import Any, Dict, List + +from sqlalchemy import ( + TIMESTAMP, + BigInteger, + Column, + ForeignKey, + Index, + String, + UniqueConstraint, +) from sqlalchemy.orm import relationship from sqlalchemy_utils import ChoiceType -from aleph.types.files import FileType -from .base import Base -import datetime as dt +from aleph.types.files import FileTag, FileType -from aleph.types.files import FileTag +from .base import Base class FilePinType(str, Enum): @@ -75,9 +82,7 @@ class FilePinDb(Base): __mapper_args__: Dict[str, Any] = { "polymorphic_on": type, } - __table_args__ = ( - UniqueConstraint("item_hash", "type"), - ) + __table_args__ = (UniqueConstraint("item_hash", "type"),) class TxFilePinDb(FilePinDb): @@ -111,7 +116,6 @@ class GracePeriodFilePinDb(FilePinDb): } - Index( "ix_file_pins_owner", MessageFilePinDb.owner, diff --git a/src/aleph/db/models/messages.py b/src/aleph/db/models/messages.py index e1cab2085..0d2f8e894 100644 --- a/src/aleph/db/models/messages.py +++ b/src/aleph/db/models/messages.py @@ -1,27 +1,27 @@ import datetime as dt -from typing import Any, Dict, List, Optional, Type, Mapping +from typing import Any, Dict, List, Mapping, Optional, Type from aleph_message.models import ( - Chain, - MessageType, - ItemType, AggregateContent, BaseContent, + Chain, ForgetContent, + InstanceContent, + ItemType, + MessageType, PostContent, ProgramContent, StoreContent, - InstanceContent, ) from pydantic import ValidationError from pydantic.error_wrappers import ErrorWrapper from sqlalchemy import ( - Column, + ARRAY, TIMESTAMP, - String, - Integer, + Column, ForeignKey, - ARRAY, + Integer, + String, Table, UniqueConstraint, ) @@ -31,7 +31,8 @@ from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.channel import Channel -from aleph.types.message_status import MessageStatus, ErrorCode +from aleph.types.message_status import ErrorCode, MessageStatus + from .base import Base from .chains import ChainTxDb from .pending_messages import PendingMessageDb diff --git a/src/aleph/db/models/peers.py b/src/aleph/db/models/peers.py index 61c1e4324..dfb5dbd27 100644 --- a/src/aleph/db/models/peers.py +++ b/src/aleph/db/models/peers.py @@ -1,7 +1,7 @@ import datetime as dt from enum import Enum -from sqlalchemy import Column, String, TIMESTAMP +from sqlalchemy import TIMESTAMP, Column, String from sqlalchemy_utils.types.choice import ChoiceType from .base import Base diff --git a/src/aleph/db/models/pending_messages.py b/src/aleph/db/models/pending_messages.py index ca0e875bd..16579463f 100644 --- a/src/aleph/db/models/pending_messages.py +++ b/src/aleph/db/models/pending_messages.py @@ -1,17 +1,17 @@ import datetime as dt -from typing import Optional, Any, Dict, Mapping +from typing import Any, Dict, Mapping, Optional -from aleph_message.models import Chain, MessageType, ItemType +from aleph_message.models import Chain, ItemType, MessageType from sqlalchemy import ( - Boolean, + TIMESTAMP, BigInteger, + Boolean, + CheckConstraint, Column, - TIMESTAMP, - String, - Integer, ForeignKey, Index, - CheckConstraint, + Integer, + String, ) from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship @@ -20,6 +20,7 @@ from aleph.schemas.pending_messages import BasePendingMessage from aleph.toolkit.timestamp import timestamp_to_datetime, utc_now from aleph.types.channel import Channel + from .base import Base from .chains import ChainTxDb diff --git a/src/aleph/db/models/posts.py b/src/aleph/db/models/posts.py index 4b2c89263..a826b5ac2 100644 --- a/src/aleph/db/models/posts.py +++ b/src/aleph/db/models/posts.py @@ -1,10 +1,11 @@ import datetime as dt from typing import Any, Optional -from sqlalchemy import Column, String, TIMESTAMP, ForeignKey +from sqlalchemy import TIMESTAMP, Column, ForeignKey, String from sqlalchemy.dialects.postgresql import JSONB from aleph.types.channel import Channel + from .base import Base diff --git a/src/aleph/db/models/vms.py b/src/aleph/db/models/vms.py index eec7d4b11..b8f6bff51 100644 --- a/src/aleph/db/models/vms.py +++ b/src/aleph/db/models/vms.py @@ -1,14 +1,15 @@ import datetime as dt -from typing import Any, Optional, Dict, List +from typing import Any, Dict, List, Optional -from aleph_message.models.execution.program import MachineType, Encoding +from aleph_message.models.execution.program import Encoding, MachineType from aleph_message.models.execution.volume import VolumePersistence -from sqlalchemy import Column, String, ForeignKey, Boolean, Integer, TIMESTAMP +from sqlalchemy import TIMESTAMP, Boolean, Column, ForeignKey, Integer, String from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.orm import relationship, declared_attr, Mapped +from sqlalchemy.orm import Mapped, declared_attr, relationship from sqlalchemy_utils import ChoiceType -from aleph.types.vms import CpuArchitecture, VmVersion, VmType +from aleph.types.vms import CpuArchitecture, VmType, VmVersion + from .base import Base diff --git a/src/aleph/exceptions.py b/src/aleph/exceptions.py index 59665fe4e..b053ded5c 100644 --- a/src/aleph/exceptions.py +++ b/src/aleph/exceptions.py @@ -1,8 +1,7 @@ from __future__ import annotations -class AlephException(Exception): - ... +class AlephException(Exception): ... class AlephStorageException(AlephException): @@ -10,15 +9,14 @@ class AlephStorageException(AlephException): Base exception class for all errors related to the storage and retrieval of Aleph messages. """ + ... -class InvalidConfigException(AlephException): - ... +class InvalidConfigException(AlephException): ... -class KeyNotFoundException(AlephException): - ... +class KeyNotFoundException(AlephException): ... class InvalidContent(AlephStorageException): @@ -27,6 +25,7 @@ class InvalidContent(AlephStorageException): * its integrity is compromised * it does not match the Aleph message specification. """ + ... @@ -35,8 +34,8 @@ class ContentCurrentlyUnavailable(AlephStorageException): The content is currently unavailable, for example because of a synchronisation issue. """ + ... -class UnknownHashError(AlephException): - ... +class UnknownHashError(AlephException): ... diff --git a/src/aleph/handlers/content/aggregate.py b/src/aleph/handlers/content/aggregate.py index efd655104..eef63d1f1 100644 --- a/src/aleph/handlers/content/aggregate.py +++ b/src/aleph/handlers/content/aggregate.py @@ -1,22 +1,23 @@ import itertools import logging -from typing import List, cast, Sequence, Set +from typing import List, Sequence, Set, cast from aleph_message.models import AggregateContent from aleph.db.accessors.aggregates import ( + count_aggregate_elements, + delete_aggregate, + delete_aggregate_element, get_aggregate_by_key, - merge_aggregate_elements, + get_aggregate_content_keys, insert_aggregate, insert_aggregate_element, + mark_aggregate_as_dirty, + merge_aggregate_elements, refresh_aggregate, update_aggregate, - count_aggregate_elements, - mark_aggregate_as_dirty, - get_aggregate_content_keys, - delete_aggregate_element, delete_aggregate, ) -from aleph.db.models import MessageDb, AggregateElementDb, AggregateDb +from aleph.db.models import AggregateDb, AggregateElementDb, MessageDb from aleph.handlers.content.content_handler import ContentHandler from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.db_session import DbSession @@ -209,7 +210,7 @@ async def process(self, session: DbSession, messages: List[MessageDb]) -> None: key=lambda m: (m.parsed_content.key, m.parsed_content.address, m.time), ) - for ((key, owner), messages_by_aggregate) in itertools.groupby( + for (key, owner), messages_by_aggregate in itertools.groupby( sorted_messages, key=lambda m: (m.parsed_content.key, m.parsed_content.address), ): diff --git a/src/aleph/handlers/content/forget.py b/src/aleph/handlers/content/forget.py index 772e6f3ff..3801ee6af 100644 --- a/src/aleph/handlers/content/forget.py +++ b/src/aleph/handlers/content/forget.py @@ -1,35 +1,29 @@ from __future__ import annotations import logging -from typing import List, cast, Sequence, Dict, Set +from typing import Dict, List, Sequence, Set, cast -from aleph_message.models import ( - MessageType, - ForgetContent, - ItemHash, -) +from aleph_message.models import ForgetContent, ItemHash, MessageType from sqlalchemy import select -from aleph.db.accessors.aggregates import ( - aggregate_exists, -) +from aleph.db.accessors.aggregates import aggregate_exists from aleph.db.accessors.messages import ( - message_exists, - get_message_status, append_to_forgotten_by, forget_message, get_message_by_item_hash, + get_message_status, + message_exists, ) -from aleph.db.models import MessageDb, AggregateElementDb +from aleph.db.models import AggregateElementDb, MessageDb from aleph.handlers.content.content_handler import ContentHandler from aleph.types.db_session import DbSession from aleph.types.message_status import ( - MessageStatus, - PermissionDenied, CannotForgetForgetMessage, ForgetTargetNotFound, - NoForgetTarget, InternalError, + MessageStatus, + NoForgetTarget, + PermissionDenied, ) logger = logging.getLogger(__name__) @@ -121,7 +115,7 @@ async def check_permissions(self, session: DbSession, message: MessageDb): ) if target_message.type == MessageType.forget: logger.warning( - f"FORGET message %s may not forget FORGET message %s", + "FORGET message %s may not forget FORGET message %s", message.item_hash, target_hash, ) diff --git a/src/aleph/handlers/content/post.py b/src/aleph/handlers/content/post.py index a3ddcafbd..044634c56 100644 --- a/src/aleph/handlers/content/post.py +++ b/src/aleph/handlers/content/post.py @@ -1,28 +1,29 @@ import logging -from typing import List, Any, Dict, Mapping, Union, Optional, Set +from typing import Any, Dict, List, Mapping, Optional, Set, Union -from aleph_message.models import PostContent, ChainRef, Chain +from aleph_message.models import Chain, ChainRef, PostContent from sqlalchemy import update from aleph.db.accessors.balances import update_balances as update_balances_db from aleph.db.accessors.posts import ( + delete_amends, + delete_post, get_matching_posts, get_original_post, - delete_post, refresh_latest_amend, - delete_amends, ) from aleph.db.models.messages import MessageDb from aleph.db.models.posts import PostDb from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.db_session import DbSession from aleph.types.message_status import ( - InvalidMessageFormat, - CannotAmendAmend, AmendTargetNotFound, - NoAmendTarget, + CannotAmendAmend, InternalError, + InvalidMessageFormat, + NoAmendTarget, ) + from .content_handler import ContentHandler LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/handlers/content/store.py b/src/aleph/handlers/content/store.py index c785bb13b..a64f0cb6d 100644 --- a/src/aleph/handlers/content/store.py +++ b/src/aleph/handlers/content/store.py @@ -11,19 +11,19 @@ from typing import List, Optional, Set import aioipfs -from aleph_message.models import ItemType, StoreContent, ItemHash +from aleph_message.models import ItemHash, ItemType, StoreContent from aleph.config import get_config from aleph.db.accessors.files import ( - insert_message_file_pin, - get_file_tag, - upsert_file_tag, delete_file_pin, - refresh_file_tag, - is_pinned_file, + get_file_tag, get_message_file_pin, - upsert_file, insert_grace_period_file_pin, + insert_message_file_pin, + is_pinned_file, + refresh_file_tag, + upsert_file, + upsert_file_tag, ) from aleph.db.models import MessageDb from aleph.exceptions import AlephStorageException, UnknownHashError @@ -33,11 +33,11 @@ from aleph.types.db_session import DbSession from aleph.types.files import FileTag, FileType from aleph.types.message_status import ( - PermissionDenied, FileUnavailable, InvalidMessageFormat, - StoreRefNotFound, + PermissionDenied, StoreCannotUpdateStoreWithRef, + StoreRefNotFound, ) from aleph.utils import item_type_from_hash diff --git a/src/aleph/handlers/content/vm.py b/src/aleph/handlers/content/vm.py index 6e8318450..2b3a4b7bd 100644 --- a/src/aleph/handlers/content/vm.py +++ b/src/aleph/handlers/content/vm.py @@ -1,54 +1,47 @@ import logging -import math -from typing import List, Set, overload, Protocol, Optional - -from aleph_message.models import ( - ProgramContent, - ExecutableContent, - InstanceContent, - MessageType, -) +from decimal import Decimal +from typing import List, Protocol, Set, overload + +from aleph_message.models import ExecutableContent, InstanceContent, ProgramContent from aleph_message.models.execution.volume import ( AbstractVolume, - ImmutableVolume, EphemeralVolume, - PersistentVolume, + ImmutableVolume, ParentVolume, + PersistentVolume, ) -from decimal import Decimal from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import ( - find_file_tags, find_file_pins, + find_file_tags, get_file_tag, get_message_file_pin, ) from aleph.db.accessors.vms import ( delete_vm, - get_program, - upsert_vm_version, delete_vm_updates, - refresh_vm_version, + get_program, is_vm_amend_allowed, + refresh_vm_version, + upsert_vm_version, ) from aleph.db.models import ( - MessageDb, CodeVolumeDb, DataVolumeDb, + EphemeralVolumeDb, ExportVolumeDb, - MachineVolumeBaseDb, ImmutableVolumeDb, - EphemeralVolumeDb, + MachineVolumeBaseDb, + MessageDb, PersistentVolumeDb, - RuntimeDb, - VmInstanceDb, ProgramDb, RootfsVolumeDb, - VmBaseDb, + RuntimeDb, StoredFileDb, - FilePinDb, + VmBaseDb, + VmInstanceDb, ) from aleph.handlers.content.content_handler import ContentHandler from aleph.services.cost import compute_cost @@ -56,14 +49,14 @@ from aleph.types.db_session import DbSession from aleph.types.files import FileTag from aleph.types.message_status import ( + InsufficientBalanceException, InternalError, InvalidMessageFormat, + VmCannotUpdateUpdate, VmRefNotFound, - VmVolumeNotFound, VmUpdateNotAllowed, - VmCannotUpdateUpdate, + VmVolumeNotFound, VmVolumeTooSmall, - InsufficientBalanceException, ) from aleph.types.vms import VmVersion @@ -79,15 +72,10 @@ def _get_vm_content(message: MessageDb) -> ExecutableContent: return content -from aleph_message.models.execution.program import ( - MachineType, - ProgramContent, -) - - @overload -def _map_content_to_db_model(item_hash: str, content: InstanceContent) -> VmInstanceDb: - ... +def _map_content_to_db_model( + item_hash: str, content: InstanceContent +) -> VmInstanceDb: ... # For some reason, mypy is not happy with the overload resolution here. diff --git a/src/aleph/handlers/message_handler.py b/src/aleph/handlers/message_handler.py index 9cdabc3ed..f24b3e678 100644 --- a/src/aleph/handlers/message_handler.py +++ b/src/aleph/handlers/message_handler.py @@ -1,11 +1,11 @@ import datetime as dt import logging -from typing import Optional, Dict, Any, Mapping +from typing import Any, Dict, Mapping, Optional import aio_pika.abc import psycopg2 import sqlalchemy.exc -from aleph_message.models import MessageType, ItemType +from aleph_message.models import ItemType, MessageType from configmanager import Config from pydantic import ValidationError from sqlalchemy import insert @@ -15,19 +15,15 @@ from aleph.db.accessors.messages import ( get_message_by_item_hash, make_confirmation_upsert_query, - make_message_upsert_query, make_message_status_upsert_query, + make_message_upsert_query, reject_new_pending_message, ) from aleph.db.accessors.pending_messages import delete_pending_message -from aleph.db.models import ( - PendingMessageDb, - MessageDb, - MessageStatusDb, -) +from aleph.db.models import MessageDb, MessageStatusDb, PendingMessageDb from aleph.exceptions import ( - InvalidContent, ContentCurrentlyUnavailable, + InvalidContent, UnknownHashError, ) from aleph.handlers.content.aggregate import AggregateMessageHandler @@ -39,15 +35,15 @@ from aleph.schemas.pending_messages import parse_message from aleph.storage import StorageService from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.files import FileType from aleph.types.message_processing_result import ProcessedMessage from aleph.types.message_status import ( InvalidMessageException, + InvalidMessageFormat, InvalidSignature, MessageContentUnavailable, MessageStatus, - InvalidMessageFormat, ) LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/jobs/__init__.py b/src/aleph/jobs/__init__.py index 160047ff4..70a436adb 100644 --- a/src/aleph/jobs/__init__.py +++ b/src/aleph/jobs/__init__.py @@ -1,13 +1,13 @@ import logging from multiprocessing import Process -from typing import Dict, List, Coroutine +from typing import Coroutine, List from aleph.jobs.fetch_pending_messages import fetch_pending_messages_subprocess from aleph.jobs.process_pending_messages import ( - pending_messages_subprocess, fetch_and_process_messages_task, + pending_messages_subprocess, ) -from aleph.jobs.process_pending_txs import pending_txs_subprocess, handle_txs_task +from aleph.jobs.process_pending_txs import handle_txs_task, pending_txs_subprocess from aleph.jobs.reconnect_ipfs import reconnect_ipfs_job from aleph.services.ipfs import IpfsService from aleph.types.db_session import DbSessionFactory @@ -42,9 +42,7 @@ def start_jobs( p2.start() p3.start() else: - tasks.append( - fetch_and_process_messages_task(config=config) - ) + tasks.append(fetch_and_process_messages_task(config=config)) tasks.append(handle_txs_task(config)) if config.ipfs.enabled.value: diff --git a/src/aleph/jobs/fetch_pending_messages.py b/src/aleph/jobs/fetch_pending_messages.py index 719469f3f..f20616281 100644 --- a/src/aleph/jobs/fetch_pending_messages.py +++ b/src/aleph/jobs/fetch_pending_messages.py @@ -4,14 +4,7 @@ import asyncio from logging import getLogger -from typing import ( - Dict, - List, - Set, - AsyncIterator, - Sequence, - NewType, -) +from typing import AsyncIterator, Dict, List, NewType, Sequence, Set import aio_pika.abc from configmanager import Config @@ -19,23 +12,23 @@ from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.accessors.pending_messages import ( - make_pending_message_fetched_statement, get_next_pending_messages, + make_pending_message_fetched_statement, ) from aleph.db.connection import make_engine, make_session_factory from aleph.db.models import MessageDb, PendingMessageDb from aleph.handlers.message_handler import MessageHandler from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService -from aleph.services.ipfs.common import make_ipfs_client from aleph.services.storage.fileystem_engine import FileSystemStorageEngine from aleph.storage import StorageService from aleph.toolkit.logging import setup_logging from aleph.toolkit.monitoring import setup_sentry from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSessionFactory -from .job_utils import prepare_loop, MessageJob, make_pending_message_queue + from ..toolkit.rabbitmq import make_mq_conn +from .job_utils import MessageJob, make_pending_message_queue, prepare_loop LOGGER = getLogger(__name__) @@ -175,9 +168,12 @@ async def fetch_messages_task(config: Config): config=config, routing_key="fetch.*", channel=mq_channel ) - async with NodeCache( - redis_host=config.redis.host.value, redis_port=config.redis.port.value - ) as node_cache, IpfsService.new(config) as ipfs_service: + async with ( + NodeCache( + redis_host=config.redis.host.value, redis_port=config.redis.port.value + ) as node_cache, + IpfsService.new(config) as ipfs_service, + ): storage_service = StorageService( storage_engine=FileSystemStorageEngine(folder=config.storage.folder.value), ipfs_service=ipfs_service, diff --git a/src/aleph/jobs/job_utils.py b/src/aleph/jobs/job_utils.py index 20c5819c9..92453cbe7 100644 --- a/src/aleph/jobs/job_utils.py +++ b/src/aleph/jobs/job_utils.py @@ -1,8 +1,7 @@ import asyncio import datetime as dt import logging -from typing import Dict, Union, Optional -from typing import Tuple +from typing import Dict, Optional, Tuple, Union import aio_pika from configmanager import Config @@ -18,9 +17,9 @@ from aleph.types.message_processing_result import RejectedMessage, WillRetryMessage from aleph.types.message_status import ( ErrorCode, - RetryMessageException, FileNotFoundException, InvalidMessageException, + RetryMessageException, ) LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/jobs/process_pending_messages.py b/src/aleph/jobs/process_pending_messages.py index 6e45a4214..60bdee06f 100644 --- a/src/aleph/jobs/process_pending_messages.py +++ b/src/aleph/jobs/process_pending_messages.py @@ -1,13 +1,10 @@ """ Job in charge of (re-) processing Aleph messages waiting in the pending queue. """ + import asyncio from logging import getLogger -from typing import ( - Dict, - AsyncIterator, - Sequence, -) +from typing import AsyncIterator, Dict, Sequence import aio_pika.abc from configmanager import Config @@ -27,10 +24,8 @@ from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSessionFactory from aleph.types.message_processing_result import MessageProcessingResult -from .job_utils import ( - prepare_loop, - MessageJob, -) + +from .job_utils import MessageJob, prepare_loop LOGGER = getLogger(__name__) @@ -153,9 +148,12 @@ async def fetch_and_process_messages_task(config: Config): engine = make_engine(config=config, application_name="aleph-process") session_factory = make_session_factory(engine) - async with NodeCache( - redis_host=config.redis.host.value, redis_port=config.redis.port.value - ) as node_cache, IpfsService.new(config) as ipfs_service: + async with ( + NodeCache( + redis_host=config.redis.host.value, redis_port=config.redis.port.value + ) as node_cache, + IpfsService.new(config) as ipfs_service, + ): storage_service = StorageService( storage_engine=FileSystemStorageEngine(folder=config.storage.folder.value), ipfs_service=ipfs_service, diff --git a/src/aleph/jobs/process_pending_txs.py b/src/aleph/jobs/process_pending_txs.py index be352a36b..f46ccb695 100644 --- a/src/aleph/jobs/process_pending_txs.py +++ b/src/aleph/jobs/process_pending_txs.py @@ -11,12 +11,11 @@ from setproctitle import setproctitle from aleph.chains.chain_data_service import ChainDataService -from aleph.db.accessors.pending_txs import get_pending_txs, delete_pending_tx +from aleph.db.accessors.pending_txs import delete_pending_tx, get_pending_txs from aleph.db.connection import make_engine, make_session_factory from aleph.db.models import PendingTxDb from aleph.handlers.message_handler import MessagePublisher from aleph.services.cache.node_cache import NodeCache -from aleph.services.ipfs.common import make_ipfs_client from aleph.services.ipfs.service import IpfsService from aleph.services.storage.fileystem_engine import FileSystemStorageEngine from aleph.storage import StorageService @@ -26,7 +25,8 @@ from aleph.toolkit.timestamp import utc_now from aleph.types.chain_sync import ChainSyncProtocol from aleph.types.db_session import DbSessionFactory -from .job_utils import MqWatcher, prepare_loop, make_pending_tx_queue + +from .job_utils import MqWatcher, make_pending_tx_queue, prepare_loop LOGGER = logging.getLogger(__name__) @@ -129,9 +129,12 @@ async def handle_txs_task(config: Config): ) pending_tx_queue = await make_pending_tx_queue(config=config, channel=mq_channel) - async with NodeCache( - redis_host=config.redis.host.value, redis_port=config.redis.port.value - ) as node_cache, IpfsService.new(config) as ipfs_service: + async with ( + NodeCache( + redis_host=config.redis.host.value, redis_port=config.redis.port.value + ) as node_cache, + IpfsService.new(config) as ipfs_service, + ): storage_service = StorageService( storage_engine=FileSystemStorageEngine(folder=config.storage.folder.value), ipfs_service=ipfs_service, diff --git a/src/aleph/network.py b/src/aleph/network.py index 69c1bd9cb..2470bb0ff 100644 --- a/src/aleph/network.py +++ b/src/aleph/network.py @@ -1,5 +1,5 @@ import logging -from typing import Coroutine, List, Any, Dict +from typing import Any, Coroutine, Dict, List from urllib.parse import unquote import aio_pika.abc @@ -55,7 +55,7 @@ async def listener_tasks( ) pending_message_exchange = await mq_channel.declare_exchange( name=config.rabbitmq.pending_message_exchange.value, - type = aio_pika.ExchangeType.TOPIC, + type=aio_pika.ExchangeType.TOPIC, auto_delete=False, ) message_publisher = MessagePublisher( diff --git a/src/aleph/schemas/api/accounts.py b/src/aleph/schemas/api/accounts.py index 3d00c03b4..ca904d3af 100644 --- a/src/aleph/schemas/api/accounts.py +++ b/src/aleph/schemas/api/accounts.py @@ -1,7 +1,8 @@ -from pydantic import BaseModel, Field +import datetime as dt from decimal import Decimal from typing import List -import datetime as dt + +from pydantic import BaseModel, Field from aleph.types.files import FileType from aleph.types.sort_order import SortOrder diff --git a/src/aleph/schemas/api/messages.py b/src/aleph/schemas/api/messages.py index 68a999fcf..a7322f7b5 100644 --- a/src/aleph/schemas/api/messages.py +++ b/src/aleph/schemas/api/messages.py @@ -1,15 +1,15 @@ import datetime as dt from typing import ( - Optional, - Generic, - TypeVar, - Literal, - List, + Annotated, Any, - Union, Dict, + Generic, + List, + Literal, Mapping, - Annotated, + Optional, + TypeVar, + Union, ) from aleph_message.models import ( @@ -17,18 +17,19 @@ BaseContent, Chain, ForgetContent, + InstanceContent, + ItemType, + MessageType, PostContent, ProgramContent, StoreContent, - InstanceContent, ) -from aleph_message.models import MessageType, ItemType from pydantic import BaseModel, Field from pydantic.generics import GenericModel import aleph.toolkit.json as aleph_json from aleph.db.models import MessageDb -from aleph.types.message_status import MessageStatus, ErrorCode +from aleph.types.message_status import ErrorCode, MessageStatus MType = TypeVar("MType", bound=MessageType) ContentType = TypeVar("ContentType", bound=BaseContent) @@ -68,14 +69,12 @@ class Config: class AggregateMessage( BaseMessage[Literal[MessageType.aggregate], AggregateContent] # type: ignore -): - ... +): ... class ForgetMessage( BaseMessage[Literal[MessageType.forget], ForgetContent] # type: ignore -): - ... +): ... class InstanceMessage(BaseMessage[Literal[MessageType.instance], InstanceContent]): # type: ignore @@ -88,14 +87,12 @@ class PostMessage(BaseMessage[Literal[MessageType.post], PostContent]): # type: class ProgramMessage( BaseMessage[Literal[MessageType.program], ProgramContent] # type: ignore -): - ... +): ... class StoreMessage( BaseMessage[Literal[MessageType.store], StoreContent] # type: ignore -): - ... +): ... MESSAGE_CLS_DICT = { @@ -125,13 +122,13 @@ def format_message(message: MessageDb) -> AlephMessage: message_type = message.type message_cls = MESSAGE_CLS_DICT[message_type] - return message_cls.from_orm(message) # type: ignore[return-value] + return message_cls.from_orm(message) def format_message_dict(message: Dict[str, Any]) -> AlephMessage: message_type = message.get("type") message_cls = MESSAGE_CLS_DICT[message_type] - return message_cls.parse_obj(message) # type: ignore[return-value] + return message_cls.parse_obj(message) class BaseMessageStatus(BaseModel): diff --git a/src/aleph/schemas/base_messages.py b/src/aleph/schemas/base_messages.py index ebb6b4503..71aa1df72 100644 --- a/src/aleph/schemas/base_messages.py +++ b/src/aleph/schemas/base_messages.py @@ -4,10 +4,9 @@ import datetime as dt from hashlib import sha256 -from typing import Optional, Generic, TypeVar, Any, Mapping, cast +from typing import Any, Generic, Mapping, Optional, TypeVar, cast -from aleph_message.models import BaseContent, Chain -from aleph_message.models import MessageType, ItemType +from aleph_message.models import BaseContent, Chain, ItemType, MessageType from pydantic import root_validator, validator from pydantic.generics import GenericModel diff --git a/src/aleph/schemas/chains/indexer_response.py b/src/aleph/schemas/chains/indexer_response.py index 929dda4f2..f78b2ba82 100644 --- a/src/aleph/schemas/chains/indexer_response.py +++ b/src/aleph/schemas/chains/indexer_response.py @@ -4,24 +4,20 @@ import datetime as dt from enum import Enum -from typing import List, Tuple, Protocol +from typing import List, Protocol, Tuple from pydantic import BaseModel, Field, validator class GenericMessageEvent(Protocol): @property - def address(self) -> str: - ... + def address(self) -> str: ... @property - def type(self) -> str: - ... + def type(self) -> str: ... @property - def content(self) -> str: - ... + def content(self) -> str: ... @property - def timestamp_seconds(self) -> float: - ... + def timestamp_seconds(self) -> float: ... class IndexerBlockchain(str, Enum): diff --git a/src/aleph/schemas/chains/sync_events.py b/src/aleph/schemas/chains/sync_events.py index 1a25b17de..ab4304847 100644 --- a/src/aleph/schemas/chains/sync_events.py +++ b/src/aleph/schemas/chains/sync_events.py @@ -1,11 +1,11 @@ -from typing import Literal, Optional, List, Union, Annotated +import datetime as dt +from typing import Annotated, List, Literal, Optional, Union -from aleph_message.models import ItemHash, ItemType, Chain, MessageType +from aleph_message.models import Chain, ItemHash, ItemType, MessageType from pydantic import BaseModel, Field, validator from aleph.types.chain_sync import ChainSyncProtocol from aleph.types.channel import Channel -import datetime as dt class OnChainMessage(BaseModel): diff --git a/src/aleph/schemas/chains/tezos_indexer_response.py b/src/aleph/schemas/chains/tezos_indexer_response.py index 403194e31..d9204d427 100644 --- a/src/aleph/schemas/chains/tezos_indexer_response.py +++ b/src/aleph/schemas/chains/tezos_indexer_response.py @@ -1,12 +1,10 @@ import datetime as dt from enum import Enum -from typing import List, Generic, TypeVar +from typing import Generic, List, TypeVar from pydantic import BaseModel, Field from pydantic.generics import GenericModel -from aleph.schemas.chains.indexer_response import GenericMessageEvent - PayloadType = TypeVar("PayloadType") diff --git a/src/aleph/schemas/message_confirmation.py b/src/aleph/schemas/message_confirmation.py index 2613b25de..41e88a1ad 100644 --- a/src/aleph/schemas/message_confirmation.py +++ b/src/aleph/schemas/message_confirmation.py @@ -3,8 +3,12 @@ class MessageConfirmation(BaseModel): - chain: Chain = Field(..., description="Chain from which the confirmation was fetched.") - height: int = Field(..., description="Block in which the confirmation was published.") + chain: Chain = Field( + ..., description="Chain from which the confirmation was fetched." + ) + height: int = Field( + ..., description="Block in which the confirmation was published." + ) hash: str = Field( ..., description="Hash of the transaction/block in which the confirmation was published.", diff --git a/src/aleph/schemas/pending_messages.py b/src/aleph/schemas/pending_messages.py index e8203eaf0..86194b3e9 100644 --- a/src/aleph/schemas/pending_messages.py +++ b/src/aleph/schemas/pending_messages.py @@ -17,23 +17,23 @@ in aleph-client. """ -from typing import Any, Literal, Generic +from typing import Any, Generic, Literal from aleph_message.models import ( AggregateContent, ForgetContent, InstanceContent, + ItemType, + MessageType, PostContent, ProgramContent, StoreContent, ) -from aleph_message.models import ItemType, MessageType -from pydantic import ValidationError -from pydantic import root_validator +from pydantic import ValidationError, root_validator import aleph.toolkit.json as aleph_json from aleph.exceptions import UnknownHashError -from aleph.schemas.base_messages import AlephBaseMessage, MType, ContentType +from aleph.schemas.base_messages import AlephBaseMessage, ContentType, MType from aleph.types.message_status import InvalidMessageFormat from aleph.utils import item_type_from_hash @@ -126,7 +126,7 @@ class PendingStoreMessage(BasePendingMessage[Literal[MessageType.store], StoreCo class PendingInlineStoreMessage(PendingStoreMessage): item_content: str - item_type: Literal[ItemType.inline] # type: ignore[valid-type] + item_type: Literal[ItemType.inline] # type: ignore[valid-type] MESSAGE_TYPE_TO_CLASS = { diff --git a/src/aleph/services/cache/materialized_views.py b/src/aleph/services/cache/materialized_views.py index 584cd60f0..796daa7f7 100644 --- a/src/aleph/services/cache/materialized_views.py +++ b/src/aleph/services/cache/materialized_views.py @@ -1,8 +1,8 @@ +import asyncio import logging from aleph.db.accessors.messages import refresh_address_stats_mat_view from aleph.types.db_session import DbSessionFactory -import asyncio LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/services/cache/node_cache.py b/src/aleph/services/cache/node_cache.py index 48c68dfe2..dbe2637b3 100644 --- a/src/aleph/services/cache/node_cache.py +++ b/src/aleph/services/cache/node_cache.py @@ -1,4 +1,4 @@ -from typing import Any, Set, Optional, List +from typing import Any, List, Optional, Set import redis.asyncio as redis_asyncio @@ -16,7 +16,6 @@ def __init__(self, redis_host: str, redis_port: int): self._redis_client: Optional[redis_asyncio.Redis] = None - @property def redis_client(self) -> redis_asyncio.Redis: if (redis_client := self._redis_client) is None: @@ -27,7 +26,6 @@ def redis_client(self) -> redis_asyncio.Redis: return redis_client - async def open(self): self._redis_client = redis_asyncio.Redis( host=self.redis_host, port=self.redis_port diff --git a/src/aleph/services/cost.py b/src/aleph/services/cost.py index da4924360..878b8edd9 100644 --- a/src/aleph/services/cost.py +++ b/src/aleph/services/cost.py @@ -6,12 +6,11 @@ from aleph_message.models.execution.volume import ImmutableVolume from aleph.db.accessors.files import get_file_tag, get_message_file_pin -from aleph.db.models import StoredFileDb, FileTagDb, MessageFilePinDb +from aleph.db.models import FileTagDb, MessageFilePinDb, StoredFileDb from aleph.toolkit.constants import GiB, MiB from aleph.types.db_session import DbSession from aleph.types.files import FileTag - MINUTE = 60 HOUR = 60 * MINUTE @@ -22,7 +21,7 @@ STORAGE_INCLUDED_PER_COMPUTE_UNIT_ON_DEMAND = Decimal("2") * GiB STORAGE_INCLUDED_PER_COMPUTE_UNIT_PERSISTENT = Decimal("20") * GiB -EXTRA_STORAGE_TOKEN_TO_HOLD = 1 / (Decimal('20') * MiB) # Hold 1 token for 20 MiB +EXTRA_STORAGE_TOKEN_TO_HOLD = 1 / (Decimal("20") * MiB) # Hold 1 token for 20 MiB EXTRA_STORAGE_PRICE_PER_HOUR = Decimal("0.000000977") EXTRA_STORAGE_PRICE_PER_SECOND = EXTRA_STORAGE_PRICE_PER_HOUR / Decimal(HOUR) diff --git a/src/aleph/services/ipfs/common.py b/src/aleph/services/ipfs/common.py index f837e3358..0fc52f645 100644 --- a/src/aleph/services/ipfs/common.py +++ b/src/aleph/services/ipfs/common.py @@ -8,7 +8,6 @@ async def get_base_url(config): return "http://{}:{}".format(config.ipfs.host.value, config.ipfs.port.value) - def make_ipfs_client(config: Config, timeout: int = 60) -> aioipfs.AsyncIPFS: host = config.ipfs.host.value port = config.ipfs.port.value diff --git a/src/aleph/services/ipfs/pubsub.py b/src/aleph/services/ipfs/pubsub.py index 4c11db44a..e62249255 100644 --- a/src/aleph/services/ipfs/pubsub.py +++ b/src/aleph/services/ipfs/pubsub.py @@ -1,10 +1,11 @@ import asyncio import logging -from .service import IpfsService from aleph.toolkit.timestamp import utc_now from aleph.types.message_status import InvalidMessageException +from .service import IpfsService + LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/services/ipfs/service.py b/src/aleph/services/ipfs/service.py index 33790db62..8dd222d24 100644 --- a/src/aleph/services/ipfs/service.py +++ b/src/aleph/services/ipfs/service.py @@ -2,7 +2,7 @@ import concurrent import json import logging -from typing import IO, Optional, Union, Dict, Self +from typing import Dict, Optional, Self, Union import aiohttp import aioipfs @@ -75,7 +75,7 @@ async def get_ipfs_content( result = None await asyncio.sleep(0.5) continue - except (asyncio.TimeoutError): + except asyncio.TimeoutError: result = None await asyncio.sleep(0.5) except ( @@ -136,7 +136,9 @@ async def _pin_add(self, cid: str, timeout: int = 30): details = "file not found" else: details = "could not fetch some blocks" - raise FileUnavailable(f"Could not pin IPFS content at this time ({details})") + raise FileUnavailable( + f"Could not pin IPFS content at this time ({details})" + ) else: # Reset the timeout counter if there is some measure of progress tick_timeout = timeout * 2 diff --git a/src/aleph/services/p2p/__init__.py b/src/aleph/services/p2p/__init__.py index bdefc49f9..32464e83d 100644 --- a/src/aleph/services/p2p/__init__.py +++ b/src/aleph/services/p2p/__init__.py @@ -4,9 +4,10 @@ from configmanager import Config from aleph.services.ipfs import IpfsService -from .manager import initialize_host from aleph.types.db_session import DbSessionFactory + from ..cache.node_cache import NodeCache +from .manager import initialize_host async def init_p2p_client(config: Config, service_name: str) -> AlephP2PServiceClient: diff --git a/src/aleph/services/p2p/http.py b/src/aleph/services/p2p/http.py index ad7a5abee..53b92d537 100644 --- a/src/aleph/services/p2p/http.py +++ b/src/aleph/services/p2p/http.py @@ -6,7 +6,7 @@ import base64 import logging from random import sample -from typing import Optional, List, Sequence +from typing import List, Optional, Sequence import aiohttp diff --git a/src/aleph/services/p2p/jobs.py b/src/aleph/services/p2p/jobs.py index 07d58be80..2203345dd 100644 --- a/src/aleph/services/p2p/jobs.py +++ b/src/aleph/services/p2p/jobs.py @@ -1,7 +1,7 @@ import asyncio import logging from dataclasses import dataclass -from typing import List, Optional +from typing import Optional from aleph_p2p_client import AlephP2PServiceClient from configmanager import Config @@ -9,9 +9,10 @@ from aleph.db.accessors.peers import get_all_addresses_by_peer_type from aleph.db.models import PeerType from aleph.types.db_session import DbSessionFactory + +from ..cache.node_cache import NodeCache from .http import api_get_request from .peers import connect_peer -from ..cache.node_cache import NodeCache @dataclass @@ -90,7 +91,9 @@ async def tidy_http_peers_job( peer_statuses = await asyncio.gather(*jobs) for peer_status in peer_statuses: - peer_in_api_servers = await node_cache.has_api_server(peer_status.peer_uri) + peer_in_api_servers = await node_cache.has_api_server( + peer_status.peer_uri + ) if peer_status.is_online: if not peer_in_api_servers: diff --git a/src/aleph/services/p2p/peers.py b/src/aleph/services/p2p/peers.py index 41ec665a0..5c7d48f5e 100644 --- a/src/aleph/services/p2p/peers.py +++ b/src/aleph/services/p2p/peers.py @@ -1,5 +1,6 @@ from aleph_p2p_client import AlephP2PServiceClient from multiaddr import Multiaddr + from aleph.toolkit.libp2p_stubs.peer.peerinfo import info_from_p2p_addr diff --git a/src/aleph/services/p2p/protocol.py b/src/aleph/services/p2p/protocol.py index 6a4f3d808..17b66daa5 100644 --- a/src/aleph/services/p2p/protocol.py +++ b/src/aleph/services/p2p/protocol.py @@ -3,7 +3,7 @@ from aleph_p2p_client import AlephP2PServiceClient -from aleph.handlers.message_handler import MessageHandler, MessagePublisher +from aleph.handlers.message_handler import MessagePublisher from aleph.network import decode_pubsub_message from aleph.toolkit.timestamp import utc_now from aleph.types.message_status import InvalidMessageException diff --git a/src/aleph/services/peers/monitor.py b/src/aleph/services/peers/monitor.py index 06d7c9201..0a6593288 100644 --- a/src/aleph/services/peers/monitor.py +++ b/src/aleph/services/peers/monitor.py @@ -53,7 +53,9 @@ async def handle_incoming_host( async def monitor_hosts_p2p( - p2p_client: AlephP2PServiceClient, session_factory: DbSessionFactory, alive_topic: str + p2p_client: AlephP2PServiceClient, + session_factory: DbSessionFactory, + alive_topic: str, ) -> None: while True: try: diff --git a/src/aleph/services/storage/engine.py b/src/aleph/services/storage/engine.py index 9b8c06663..ed3ae77d6 100644 --- a/src/aleph/services/storage/engine.py +++ b/src/aleph/services/storage/engine.py @@ -1,21 +1,16 @@ import abc -import datetime as dt from typing import Optional class StorageEngine(abc.ABC): @abc.abstractmethod - async def read(self, filename: str) -> Optional[bytes]: - ... + async def read(self, filename: str) -> Optional[bytes]: ... @abc.abstractmethod - async def write(self, filename: str, content: bytes): - ... + async def write(self, filename: str, content: bytes): ... @abc.abstractmethod - async def delete(self, filename: str): - ... + async def delete(self, filename: str): ... @abc.abstractmethod - async def exists(self, filename: str) -> bool: - ... + async def exists(self, filename: str) -> bool: ... diff --git a/src/aleph/services/storage/garbage_collector.py b/src/aleph/services/storage/garbage_collector.py index 9c90532e3..eae578e0c 100644 --- a/src/aleph/services/storage/garbage_collector.py +++ b/src/aleph/services/storage/garbage_collector.py @@ -4,14 +4,11 @@ from aioipfs import NotPinnedError from aioipfs.api import RepoAPI -from aleph_message.models import ItemType, ItemHash +from aleph_message.models import ItemHash, ItemType from configmanager import Config -from aleph.db.accessors.files import ( - get_unpinned_files, - delete_file as delete_file_db, - delete_grace_period_file_pins, -) +from aleph.db.accessors.files import delete_file as delete_file_db +from aleph.db.accessors.files import delete_grace_period_file_pins, get_unpinned_files from aleph.storage import StorageService from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSessionFactory @@ -43,7 +40,7 @@ async def _delete_from_ipfs(self, file_hash: ItemHash): LOGGER.debug("Deleting %s from local storage") await self._delete_from_local_storage(file_hash) - LOGGER.debug(f"Removed from IPFS: %s", file_hash) + LOGGER.debug("Removed from IPFS: %s", file_hash) async def _delete_from_local_storage(self, file_hash: ItemHash): LOGGER.debug(f"Removing from local storage: {file_hash}") diff --git a/src/aleph/services/utils.py b/src/aleph/services/utils.py index 9d5472861..6eb34528c 100644 --- a/src/aleph/services/utils.py +++ b/src/aleph/services/utils.py @@ -43,6 +43,6 @@ async def get_IP() -> str: """Get the public IPv4 of this system.""" try: return await get_ip4_from_service() - except Exception as error: + except Exception: logging.exception("Error when fetching IPv4 from service") return get_ip4_from_socket() diff --git a/src/aleph/storage.py b/src/aleph/storage.py index 58b8f153b..62f4075b2 100644 --- a/src/aleph/storage.py +++ b/src/aleph/storage.py @@ -1,10 +1,11 @@ """ Storage module for Aleph. Basically manages the IPFS storage. """ + import asyncio import logging from hashlib import sha256 -from typing import Any, IO, Optional, cast, Final +from typing import Any, Final, Optional, cast from aleph_message.models import ItemType @@ -12,8 +13,8 @@ from aleph.config import get_config from aleph.db.accessors.files import upsert_file from aleph.db.models import PendingMessageDb -from aleph.exceptions import InvalidContent, ContentCurrentlyUnavailable -from aleph.schemas.message_content import ContentSource, RawContent, MessageContent +from aleph.exceptions import ContentCurrentlyUnavailable, InvalidContent +from aleph.schemas.message_content import ContentSource, MessageContent, RawContent from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService from aleph.services.ipfs.common import get_cid_version @@ -39,7 +40,7 @@ def check_for_u0000(item_content: aleph_json.SerializedJsonInput): contains_u0000 = U0000_BYTES in item_content if contains_u0000: - error_msg = f"Unsupported character in message: \\u0000" + error_msg = "Unsupported character in message: \\u0000" LOGGER.warning(error_msg) raise InvalidContent(error_msg) @@ -127,7 +128,7 @@ async def _compute_content_hash_ipfs( return computed_hash except asyncio.TimeoutError: - LOGGER.warning(f"Timeout while computing IPFS hash.") + LOGGER.warning("Timeout while computing IPFS hash.") return None @staticmethod diff --git a/src/aleph/toolkit/exceptions.py b/src/aleph/toolkit/exceptions.py index 7f394a3ba..82636f93f 100644 --- a/src/aleph/toolkit/exceptions.py +++ b/src/aleph/toolkit/exceptions.py @@ -5,7 +5,7 @@ @contextmanager def ignore_exceptions( *exceptions: Type[BaseException], - on_error: Optional[Callable[[BaseException], None]] = None + on_error: Optional[Callable[[BaseException], None]] = None, ): try: yield diff --git a/src/aleph/toolkit/json.py b/src/aleph/toolkit/json.py index 6dca0b32d..4c4ac2210 100644 --- a/src/aleph/toolkit/json.py +++ b/src/aleph/toolkit/json.py @@ -3,9 +3,9 @@ Makes swapping between JSON implementations easier. """ +from typing import IO, Any, Union import orjson -from typing import Any, IO, Union # The actual type of serialized JSON as returned by the JSON serializer. SerializedJson = bytes diff --git a/src/aleph/toolkit/libp2p_stubs/crypto/keys.py b/src/aleph/toolkit/libp2p_stubs/crypto/keys.py index 21b481e74..d68382a13 100644 --- a/src/aleph/toolkit/libp2p_stubs/crypto/keys.py +++ b/src/aleph/toolkit/libp2p_stubs/crypto/keys.py @@ -62,18 +62,16 @@ class PrivateKey(Key): """A ``PrivateKey`` represents a cryptographic private key.""" @abstractmethod - def sign(self, data: bytes) -> bytes: - ... + def sign(self, data: bytes) -> bytes: ... @abstractmethod - def get_public_key(self) -> PublicKey: - ... + def get_public_key(self) -> PublicKey: ... def _serialize_to_protobuf(self) -> protobuf.PrivateKey: """Return the protobuf representation of this ``Key``.""" key_type = self.get_type().value data = self.to_bytes() - protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) # type: ignore + protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) # type: ignore return protobuf_key def serialize(self) -> bytes: diff --git a/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.py b/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.py index 6e8ae6eeb..6f7084757 100644 --- a/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.py +++ b/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.py @@ -2,22 +2,23 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: crypto.proto """Generated protocol buffer code.""" -from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0c\x63rypto.proto\x12\tcrypto.pb"?\n\tPublicKey\x12$\n\x08key_type\x18\x01 \x02(\x0e\x32\x12.crypto.pb.KeyType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c"@\n\nPrivateKey\x12$\n\x08key_type\x18\x01 \x02(\x0e\x32\x12.crypto.pb.KeyType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c*9\n\x07KeyType\x12\x07\n\x03RSA\x10\x00\x12\x0b\n\x07\x45\x64\x32\x35\x35\x31\x39\x10\x01\x12\r\n\tSecp256k1\x10\x02\x12\t\n\x05\x45\x43\x44SA\x10\x03' +) - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63rypto.proto\x12\tcrypto.pb\"?\n\tPublicKey\x12$\n\x08key_type\x18\x01 \x02(\x0e\x32\x12.crypto.pb.KeyType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c\"@\n\nPrivateKey\x12$\n\x08key_type\x18\x01 \x02(\x0e\x32\x12.crypto.pb.KeyType\x12\x0c\n\x04\x64\x61ta\x18\x02 \x02(\x0c*9\n\x07KeyType\x12\x07\n\x03RSA\x10\x00\x12\x0b\n\x07\x45\x64\x32\x35\x35\x31\x39\x10\x01\x12\r\n\tSecp256k1\x10\x02\x12\t\n\x05\x45\x43\x44SA\x10\x03') - -_KEYTYPE = DESCRIPTOR.enum_types_by_name['KeyType'] +_KEYTYPE = DESCRIPTOR.enum_types_by_name["KeyType"] KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE) RSA = 0 Ed25519 = 1 @@ -25,29 +26,37 @@ ECDSA = 3 -_PUBLICKEY = DESCRIPTOR.message_types_by_name['PublicKey'] -_PRIVATEKEY = DESCRIPTOR.message_types_by_name['PrivateKey'] -PublicKey = _reflection.GeneratedProtocolMessageType('PublicKey', (_message.Message,), { - 'DESCRIPTOR' : _PUBLICKEY, - '__module__' : 'crypto_pb2' - # @@protoc_insertion_point(class_scope:crypto.pb.PublicKey) - }) +_PUBLICKEY = DESCRIPTOR.message_types_by_name["PublicKey"] +_PRIVATEKEY = DESCRIPTOR.message_types_by_name["PrivateKey"] +PublicKey = _reflection.GeneratedProtocolMessageType( + "PublicKey", + (_message.Message,), + { + "DESCRIPTOR": _PUBLICKEY, + "__module__": "crypto_pb2", + # @@protoc_insertion_point(class_scope:crypto.pb.PublicKey) + }, +) _sym_db.RegisterMessage(PublicKey) -PrivateKey = _reflection.GeneratedProtocolMessageType('PrivateKey', (_message.Message,), { - 'DESCRIPTOR' : _PRIVATEKEY, - '__module__' : 'crypto_pb2' - # @@protoc_insertion_point(class_scope:crypto.pb.PrivateKey) - }) +PrivateKey = _reflection.GeneratedProtocolMessageType( + "PrivateKey", + (_message.Message,), + { + "DESCRIPTOR": _PRIVATEKEY, + "__module__": "crypto_pb2", + # @@protoc_insertion_point(class_scope:crypto.pb.PrivateKey) + }, +) _sym_db.RegisterMessage(PrivateKey) -if _descriptor._USE_C_DESCRIPTORS == False: +if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._options = None - _KEYTYPE._serialized_start=158 - _KEYTYPE._serialized_end=215 - _PUBLICKEY._serialized_start=27 - _PUBLICKEY._serialized_end=90 - _PRIVATEKEY._serialized_start=92 - _PRIVATEKEY._serialized_end=156 + DESCRIPTOR._options = None + _KEYTYPE._serialized_start = 158 + _KEYTYPE._serialized_end = 215 + _PUBLICKEY._serialized_start = 27 + _PUBLICKEY._serialized_end = 90 + _PRIVATEKEY._serialized_start = 92 + _PRIVATEKEY._serialized_end = 156 # @@protoc_insertion_point(module_scope) diff --git a/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.pyi b/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.pyi index 825b44485..6f5d91c16 100644 --- a/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.pyi +++ b/src/aleph/toolkit/libp2p_stubs/crypto/pb/crypto_pb2.pyi @@ -2,6 +2,7 @@ @generated by mypy-protobuf. Do not edit manually! isort:skip_file """ + import builtins import google.protobuf.descriptor import google.protobuf.internal.enum_type_wrapper @@ -12,14 +13,19 @@ import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor class _KeyType: - ValueType = typing.NewType('ValueType', builtins.int) + ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType -class _KeyTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_KeyType.ValueType], builtins.type): + +class _KeyTypeEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_KeyType.ValueType], + builtins.type, +): DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor RSA: _KeyType.ValueType # 0 Ed25519: _KeyType.ValueType # 1 Secp256k1: _KeyType.ValueType # 2 ECDSA: _KeyType.ValueType # 3 + class KeyType(_KeyType, metaclass=_KeyTypeEnumTypeWrapper): pass @@ -29,20 +35,27 @@ Secp256k1: KeyType.ValueType # 2 ECDSA: KeyType.ValueType # 3 global___KeyType = KeyType - class PublicKey(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor KEY_TYPE_FIELD_NUMBER: builtins.int DATA_FIELD_NUMBER: builtins.int key_type: global___KeyType.ValueType data: builtins.bytes - def __init__(self, + def __init__( + self, *, key_type: typing.Optional[global___KeyType.ValueType] = ..., data: typing.Optional[builtins.bytes] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["data",b"data","key_type",b"key_type"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["data",b"data","key_type",b"key_type"]) -> None: ... + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["data", b"data", "key_type", b"key_type"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["data", b"data", "key_type", b"key_type"], + ) -> None: ... + global___PublicKey = PublicKey class PrivateKey(google.protobuf.message.Message): @@ -51,11 +64,19 @@ class PrivateKey(google.protobuf.message.Message): DATA_FIELD_NUMBER: builtins.int key_type: global___KeyType.ValueType data: builtins.bytes - def __init__(self, + def __init__( + self, *, key_type: typing.Optional[global___KeyType.ValueType] = ..., data: typing.Optional[builtins.bytes] = ..., - ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["data",b"data","key_type",b"key_type"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["data",b"data","key_type",b"key_type"]) -> None: ... + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["data", b"data", "key_type", b"key_type"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["data", b"data", "key_type", b"key_type"], + ) -> None: ... + global___PrivateKey = PrivateKey diff --git a/src/aleph/toolkit/libp2p_stubs/crypto/rsa.py b/src/aleph/toolkit/libp2p_stubs/crypto/rsa.py index 6e04bc86c..ea1a469b9 100644 --- a/src/aleph/toolkit/libp2p_stubs/crypto/rsa.py +++ b/src/aleph/toolkit/libp2p_stubs/crypto/rsa.py @@ -1,5 +1,5 @@ -from Crypto.Hash import SHA256 import Crypto.PublicKey.RSA as RSA +from Crypto.Hash import SHA256 from Crypto.PublicKey.RSA import RsaKey from Crypto.Signature import pkcs1_15 diff --git a/src/aleph/toolkit/libp2p_stubs/peer/id.py b/src/aleph/toolkit/libp2p_stubs/peer/id.py index 4065ab95f..1f6025739 100644 --- a/src/aleph/toolkit/libp2p_stubs/peer/id.py +++ b/src/aleph/toolkit/libp2p_stubs/peer/id.py @@ -1,5 +1,5 @@ import hashlib -from typing import Union +from typing import Optional, Union import base58 import multihash @@ -35,8 +35,8 @@ def digest(self) -> bytes: class ID: _bytes: bytes - _xor_id: int = None - _b58_str: str = None + _xor_id: Optional[int] = None + _b58_str: Optional[str] = None def __init__(self, peer_id_bytes: bytes) -> None: self._bytes = peer_id_bytes @@ -51,7 +51,7 @@ def to_bytes(self) -> bytes: return self._bytes def to_base58(self) -> str: - if not self._b58_str: + if self._b58_str is None: self._b58_str = base58.b58encode(self._bytes).decode() return self._b58_str diff --git a/src/aleph/toolkit/logging.py b/src/aleph/toolkit/logging.py index 44daf2873..728a65449 100644 --- a/src/aleph/toolkit/logging.py +++ b/src/aleph/toolkit/logging.py @@ -1,7 +1,7 @@ import logging import sys -from typing import Any, Dict, Optional from logging.handlers import RotatingFileHandler +from typing import Any, Dict, Optional def setup_logging( diff --git a/src/aleph/toolkit/range.py b/src/aleph/toolkit/range.py index fe3da12e1..23d2ebe26 100644 --- a/src/aleph/toolkit/range.py +++ b/src/aleph/toolkit/range.py @@ -1,5 +1,4 @@ -from typing import Generic, List, TypeVar, Any, Callable, Protocol -import re +from typing import Any, Callable, Generic, List, Protocol, TypeVar class Comparable(Protocol): diff --git a/src/aleph/toolkit/shield.py b/src/aleph/toolkit/shield.py index 7c5806224..60fcb7bec 100644 --- a/src/aleph/toolkit/shield.py +++ b/src/aleph/toolkit/shield.py @@ -6,6 +6,7 @@ def shielded(func): """ Protects a coroutine from cancellation. """ + @wraps(func) async def wrapped(*args, **kwargs): return await asyncio.shield(func(*args, **kwargs)) diff --git a/src/aleph/toolkit/split.py b/src/aleph/toolkit/split.py index be892811a..6790d5685 100644 --- a/src/aleph/toolkit/split.py +++ b/src/aleph/toolkit/split.py @@ -1,10 +1,11 @@ from typing import Callable, Iterable, List, Tuple, TypeVar - T = TypeVar("T") -def split_iterable(iterable: Iterable[T], cond: Callable[[T], bool]) -> Tuple[List[T], List[T]]: +def split_iterable( + iterable: Iterable[T], cond: Callable[[T], bool] +) -> Tuple[List[T], List[T]]: matches = [] others = [] diff --git a/src/aleph/toolkit/timestamp.py b/src/aleph/toolkit/timestamp.py index 38b7e5665..8fae474c5 100644 --- a/src/aleph/toolkit/timestamp.py +++ b/src/aleph/toolkit/timestamp.py @@ -1,7 +1,7 @@ +import datetime as dt from typing import Optional, Union import pytz -import datetime as dt def timestamp_to_datetime(timestamp: float) -> dt.datetime: diff --git a/src/aleph/types/message_processing_result.py b/src/aleph/types/message_processing_result.py index 9a386f143..705d6146d 100644 --- a/src/aleph/types/message_processing_result.py +++ b/src/aleph/types/message_processing_result.py @@ -1,11 +1,8 @@ from typing import Any, Dict, Protocol -from aleph.db.models import PendingMessageDb, MessageDb +from aleph.db.models import MessageDb, PendingMessageDb from aleph.schemas.api.messages import format_message -from aleph.types.message_status import ( - ErrorCode, - MessageProcessingStatus, -) +from aleph.types.message_status import ErrorCode, MessageProcessingStatus class MessageProcessingResult(Protocol): @@ -33,7 +30,10 @@ def item_hash(self) -> str: return self.message.item_hash def to_dict(self) -> Dict[str, Any]: - return {"status": self.status.value, "message": format_message(self.message).dict()} + return { + "status": self.status.value, + "message": format_message(self.message).dict(), + } class FailedMessage(MessageProcessingResult): diff --git a/src/aleph/types/message_status.py b/src/aleph/types/message_status.py index 1d92e46ab..dc8b5090f 100644 --- a/src/aleph/types/message_status.py +++ b/src/aleph/types/message_status.py @@ -1,6 +1,6 @@ -from enum import Enum, IntEnum -from typing import Optional, Any, Dict, Sequence, Union from decimal import Decimal +from enum import Enum, IntEnum +from typing import Any, Dict, Optional, Sequence, Union class MessageStatus(str, Enum): diff --git a/src/aleph/types/protocol.py b/src/aleph/types/protocol.py index faad5d58e..7f590ca09 100644 --- a/src/aleph/types/protocol.py +++ b/src/aleph/types/protocol.py @@ -5,5 +5,6 @@ class Protocol(str, Enum): """P2P Protocol""" + IPFS = "ipfs" P2P = "p2p" diff --git a/src/aleph/web/controllers/accounts.py b/src/aleph/web/controllers/accounts.py index 687275641..141b08285 100644 --- a/src/aleph/web/controllers/accounts.py +++ b/src/aleph/web/controllers/accounts.py @@ -8,15 +8,12 @@ from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address -from aleph.db.accessors.files import ( - get_address_files_for_api, - get_address_files_stats, -) +from aleph.db.accessors.files import get_address_files_for_api, get_address_files_stats from aleph.db.accessors.messages import get_message_stats_by_address from aleph.schemas.api.accounts import ( GetAccountBalanceResponse, - GetAccountFilesResponse, GetAccountFilesQueryParams, + GetAccountFilesResponse, GetAccountFilesResponseItem, ) from aleph.types.db_session import DbSessionFactory diff --git a/src/aleph/web/controllers/aggregates.py b/src/aleph/web/controllers/aggregates.py index a40317daf..09b25b0a6 100644 --- a/src/aleph/web/controllers/aggregates.py +++ b/src/aleph/web/controllers/aggregates.py @@ -3,11 +3,12 @@ from typing import Dict, List, Optional from aiohttp import web -from aleph.db.accessors.aggregates import get_aggregates_by_owner, refresh_aggregate -from aleph.db.models import AggregateDb from pydantic import BaseModel, ValidationError, validator from sqlalchemy import select +from aleph.db.accessors.aggregates import get_aggregates_by_owner, refresh_aggregate +from aleph.db.models import AggregateDb + from .utils import LIST_FIELD_SEPARATOR LOGGER = logging.getLogger(__name__) @@ -60,8 +61,10 @@ async def address_aggregate(request: web.Request) -> web.Response: aggregates = list( get_aggregates_by_owner( - session=session, owner=address, - with_info=query_params.with_info, keys=query_params.keys + session=session, + owner=address, + with_info=query_params.with_info, + keys=query_params.keys, ) ) diff --git a/src/aleph/web/controllers/app_state_getters.py b/src/aleph/web/controllers/app_state_getters.py index 27b75a624..1277388e9 100644 --- a/src/aleph/web/controllers/app_state_getters.py +++ b/src/aleph/web/controllers/app_state_getters.py @@ -3,15 +3,15 @@ This module provides an abstraction layer over the dictionary keys used to address these objects. """ + import logging -from typing import Optional, cast, TypeVar +from typing import Optional, TypeVar, cast import aio_pika.abc from aiohttp import web from aleph_p2p_client import AlephP2PServiceClient from configmanager import Config -from aleph.chains.connector import ChainConnector from aleph.chains.signature_verifier import SignatureVerifier from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService diff --git a/src/aleph/web/controllers/channels.py b/src/aleph/web/controllers/channels.py index b7c4b7c09..47e65947d 100644 --- a/src/aleph/web/controllers/channels.py +++ b/src/aleph/web/controllers/channels.py @@ -1,6 +1,6 @@ from typing import List -from aiocache import cached, SimpleMemoryCache +from aiocache import SimpleMemoryCache, cached from aiohttp import web from aleph.db.accessors.messages import get_distinct_channels diff --git a/src/aleph/web/controllers/ipfs.py b/src/aleph/web/controllers/ipfs.py index 061da6dc9..3e38f9150 100644 --- a/src/aleph/web/controllers/ipfs.py +++ b/src/aleph/web/controllers/ipfs.py @@ -6,11 +6,11 @@ from aleph.db.accessors.files import upsert_file from aleph.types.files import FileType from aleph.web.controllers.app_state_getters import ( + get_config_from_request, get_ipfs_service_from_request, get_session_factory_from_request, - get_config_from_request, ) -from aleph.web.controllers.utils import file_field_to_io, add_grace_period_for_file +from aleph.web.controllers.utils import add_grace_period_for_file async def ipfs_add_file(request: web.Request): @@ -53,7 +53,9 @@ async def ipfs_add_file(request: web.Request): # IPFS add returns the cumulative size and not the real file size. # We need the real file size here. - stats = await asyncio.wait_for(ipfs_service.ipfs_client.files.stat(f"/ipfs/{cid}"), 5) + stats = await asyncio.wait_for( + ipfs_service.ipfs_client.files.stat(f"/ipfs/{cid}"), 5 + ) size = stats["Size"] with session_factory() as session: diff --git a/src/aleph/web/controllers/messages.py b/src/aleph/web/controllers/messages.py index af580ea29..e3c1ded2d 100644 --- a/src/aleph/web/controllers/messages.py +++ b/src/aleph/web/controllers/messages.py @@ -3,8 +3,11 @@ import aio_pika.abc import aiohttp.web_ws -import aleph.toolkit.json as aleph_json from aiohttp import WSMsgType, web +from aleph_message.models import Chain, ItemHash, MessageType +from pydantic import BaseModel, Field, ValidationError, root_validator, validator + +import aleph.toolkit.json as aleph_json from aleph.db.accessors.messages import ( count_matching_messages, get_forgotten_message, @@ -42,8 +45,6 @@ LIST_FIELD_SEPARATOR, mq_make_aleph_message_topic_queue, ) -from aleph_message.models import Chain, ItemHash, MessageType -from pydantic import BaseModel, Field, ValidationError, root_validator, validator LOGGER = logging.getLogger(__name__) @@ -300,7 +301,7 @@ def message_matches_filters( if user_filters := getattr(query_params, query_field): if not isinstance(user_filters, list): user_filters = [user_filters] - if not getattr(message, message_field) in user_filters: + if getattr(message, message_field) not in user_filters: return False # Process filters on content and content.content diff --git a/src/aleph/web/controllers/metrics.py b/src/aleph/web/controllers/metrics.py index ec850b72f..bd812c706 100644 --- a/src/aleph/web/controllers/metrics.py +++ b/src/aleph/web/controllers/metrics.py @@ -1,14 +1,14 @@ import asyncio import json import platform -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass from logging import getLogger from typing import Dict, Optional from urllib.parse import urljoin import aiohttp from aiocache import cached -from aleph_message.models import MessageType, Chain +from aleph_message.models import Chain from dataclasses_json import DataClassJsonMixin from requests import HTTPError from web3 import Web3 @@ -16,7 +16,7 @@ from aleph import __version__ from aleph.config import get_config from aleph.db.accessors.chains import get_last_height -from aleph.db.models import PeerDb, MessageDb, FilePinDb, PendingMessageDb, PendingTxDb +from aleph.db.models import FilePinDb, MessageDb, PeerDb, PendingMessageDb, PendingTxDb from aleph.services.cache.node_cache import NodeCache from aleph.types.chain_sync import ChainEventType from aleph.types.db_session import DbSession diff --git a/src/aleph/web/controllers/p2p.py b/src/aleph/web/controllers/p2p.py index a1b308e0c..88939d724 100644 --- a/src/aleph/web/controllers/p2p.py +++ b/src/aleph/web/controllers/p2p.py @@ -1,7 +1,7 @@ import asyncio import json import logging -from typing import Dict, cast, Optional, Any, List, Union +from typing import Any, Dict, List, Optional, Union, cast from aiohttp import web from aleph_p2p_client import AlephP2PServiceClient @@ -18,10 +18,10 @@ get_p2p_client_from_request, ) from aleph.web.controllers.utils import ( - validate_message_dict, - broadcast_and_process_message, PublicationStatus, + broadcast_and_process_message, broadcast_status_to_http_status, + validate_message_dict, ) LOGGER = logging.getLogger(__name__) diff --git a/src/aleph/web/controllers/posts.py b/src/aleph/web/controllers/posts.py index 76fe4bf47..773d7f6df 100644 --- a/src/aleph/web/controllers/posts.py +++ b/src/aleph/web/controllers/posts.py @@ -1,6 +1,10 @@ from typing import Any, Dict, List, Optional from aiohttp import web +from aleph_message.models import ItemHash +from pydantic import BaseModel, Field, ValidationError, root_validator, validator +from sqlalchemy import select + from aleph.db.accessors.posts import ( MergedPost, MergedPostV0, @@ -19,9 +23,6 @@ cond_output, get_path_page, ) -from aleph_message.models import ItemHash -from pydantic import BaseModel, Field, ValidationError, root_validator, validator -from sqlalchemy import select class PostQueryParams(BaseModel): diff --git a/src/aleph/web/controllers/prices.py b/src/aleph/web/controllers/prices.py index 913e11cba..b22df61a8 100644 --- a/src/aleph/web/controllers/prices.py +++ b/src/aleph/web/controllers/prices.py @@ -9,7 +9,7 @@ from dataclasses_json import DataClassJsonMixin from aleph.db.accessors.messages import get_message_by_item_hash, get_message_status -from aleph.db.models import MessageDb, MessageStatusDb +from aleph.db.models import MessageDb from aleph.services.cost import compute_cost, compute_flow_cost from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.message_status import MessageStatus @@ -87,5 +87,9 @@ async def message_price(request: web.Request): else: required_tokens = compute_cost(session=session, content=content) - return web.json_response({"required_tokens": float(required_tokens), - "payment_type": content.payment.type if content.payment else None}) + return web.json_response( + { + "required_tokens": float(required_tokens), + "payment_type": content.payment.type if content.payment else None, + } + ) diff --git a/src/aleph/web/controllers/routes.py b/src/aleph/web/controllers/routes.py index 7ab22728a..c3345615f 100644 --- a/src/aleph/web/controllers/routes.py +++ b/src/aleph/web/controllers/routes.py @@ -1,5 +1,5 @@ -from aiohttp import web import pkg_resources +from aiohttp import web from aleph.web.controllers import ( accounts, diff --git a/src/aleph/web/controllers/storage.py b/src/aleph/web/controllers/storage.py index ac90d8ca4..26201a447 100644 --- a/src/aleph/web/controllers/storage.py +++ b/src/aleph/web/controllers/storage.py @@ -11,6 +11,10 @@ import pydantic from aiohttp import BodyPartReader, web from aiohttp.web_request import FileField +from aleph_message.models import ItemType, StoreContent +from mypy.dmypy_server import MiB +from pydantic import ValidationError + from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.accessors.balances import get_total_balance from aleph.db.accessors.cost import get_total_cost_for_address @@ -24,7 +28,7 @@ from aleph.storage import StorageService from aleph.types.db_session import DbSession from aleph.types.message_status import InvalidSignature -from aleph.utils import get_sha256, item_type_from_hash, run_in_executor +from aleph.utils import item_type_from_hash, run_in_executor from aleph.web.controllers.app_state_getters import ( get_config_from_request, get_mq_channel_from_request, @@ -38,9 +42,6 @@ broadcast_status_to_http_status, mq_make_aleph_message_topic_queue, ) -from aleph_message.models import ItemType, StoreContent -from mypy.dmypy_server import MiB -from pydantic import ValidationError logger = logging.getLogger(__name__) @@ -96,7 +97,7 @@ async def add_storage_json_controller(request: web.Request): async def _verify_message_signature( - pending_message: BasePendingMessage, signature_verifier: SignatureVerifier + pending_message: BasePendingMessage, signature_verifier: SignatureVerifier ) -> None: try: await signature_verifier.verify_signature(pending_message) @@ -130,7 +131,7 @@ def __init__(self, max_size: int): async def open_temp_file(self): if not self._temp_file_path: raise ValueError("File content has not been validated and read yet.") - self._temp_file = await aiofiles.open(self._temp_file_path, 'rb') + self._temp_file = await aiofiles.open(self._temp_file_path, "rb") return self._temp_file async def close_temp_file(self): @@ -158,11 +159,11 @@ async def read_and_validate(self): # it would be ideal to uses aiofiles.tempfile.NamedTemporaryFile but it # doesn't seems to be able to support our current workflow - temp_file = tempfile.NamedTemporaryFile('w+b', delete=False) + temp_file = tempfile.NamedTemporaryFile("w+b", delete=False) self._temp_file_path = temp_file.name temp_file.close() - async with aiofiles.open(self._temp_file_path, 'w+b') as f: + async with aiofiles.open(self._temp_file_path, "w+b") as f: async for chunk in self._read_chunks(chunk_size): total_read += len(chunk) if total_read > self.max_size: @@ -206,12 +207,12 @@ async def _read_chunks(self, chunk_size): async def _check_and_add_file( - session: DbSession, - signature_verifier: SignatureVerifier, - storage_service: StorageService, - message: Optional[PendingStoreMessage], - uploaded_file: UploadedFile, - grace_period: int, + session: DbSession, + signature_verifier: SignatureVerifier, + storage_service: StorageService, + message: Optional[PendingStoreMessage], + uploaded_file: UploadedFile, + grace_period: int, ) -> str: file_hash = uploaded_file.get_hash() # Perform authentication and balance checks @@ -246,12 +247,12 @@ async def _check_and_add_file( elif isinstance(file_content, str): file_bytes = file_content.encode("utf-8") else: - raise web.HTTPUnprocessableEntity(reason=f"Invalid file content type, got {type(file_content)}") + raise web.HTTPUnprocessableEntity( + reason=f"Invalid file content type, got {type(file_content)}" + ) await storage_service.add_file_content_to_local_storage( - session=session, - file_content=file_bytes, - file_hash=file_hash + session=session, file_content=file_bytes, file_hash=file_hash ) await uploaded_file.cleanup() @@ -264,9 +265,9 @@ async def _check_and_add_file( async def _make_mq_queue( - request: web.Request, - sync: bool, - routing_key: Optional[str] = None, + request: web.Request, + sync: bool, + routing_key: Optional[str] = None, ) -> Optional[aio_pika.abc.AbstractQueue]: if not sync: return None @@ -291,17 +292,21 @@ async def storage_add_file(request: web.Request): if request.content_type == "multipart/form-data": reader = await request.multipart() async for part in reader: - if part.name == 'file': + if part.name == "file": uploaded_file = MultipartUploadedFile(part, MAX_FILE_SIZE) await uploaded_file.read_and_validate() - elif part.name == 'metadata': + elif part.name == "metadata": metadata = await part.read(decode=True) else: - uploaded_file = RawUploadedFile(request=request, max_size=MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE) + uploaded_file = RawUploadedFile( + request=request, max_size=MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE + ) await uploaded_file.read_and_validate() if uploaded_file is None: - raise web.HTTPBadRequest(reason="File should be sent as FormData or Raw Upload") + raise web.HTTPBadRequest( + reason="File should be sent as FormData or Raw Upload" + ) max_upload_size = ( MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE if not metadata else MAX_FILE_SIZE diff --git a/src/aleph/web/controllers/utils.py b/src/aleph/web/controllers/utils.py index ce1d13b82..6cbcc0ebb 100644 --- a/src/aleph/web/controllers/utils.py +++ b/src/aleph/web/controllers/utils.py @@ -4,8 +4,7 @@ import logging from io import BytesIO, StringIO from math import ceil -from typing import Optional, Any, Mapping, List, Union, Dict -from typing import overload +from typing import Any, Dict, List, Mapping, Optional, Union, overload import aio_pika import aio_pika.abc @@ -18,7 +17,7 @@ import aleph.toolkit.json as aleph_json from aleph.db.accessors.files import insert_grace_period_file_pin -from aleph.schemas.pending_messages import parse_message, BasePendingMessage +from aleph.schemas.pending_messages import BasePendingMessage, parse_message from aleph.services.ipfs import IpfsService from aleph.services.p2p.pubsub import publish as pub_p2p from aleph.toolkit.shield import shielded @@ -26,15 +25,15 @@ from aleph.types.db_session import DbSession from aleph.types.message_status import ( InvalidMessageException, - MessageStatus, MessageProcessingStatus, + MessageStatus, ) from aleph.types.protocol import Protocol from aleph.web.controllers.app_state_getters import ( get_config_from_request, get_ipfs_service_from_request, - get_p2p_client_from_request, get_mq_channel_from_request, + get_p2p_client_from_request, ) DEFAULT_MESSAGES_PER_PAGE = 20 @@ -43,18 +42,15 @@ @overload -def file_field_to_io(multi_dict: bytes) -> BytesIO: - ... +def file_field_to_io(multi_dict: bytes) -> BytesIO: ... @overload -def file_field_to_io(multi_dict: str) -> StringIO: - ... +def file_field_to_io(multi_dict: str) -> StringIO: ... # type: ignore[misc] @overload -def file_field_to_io(multi_dict: FileField) -> BytesIO: - ... +def file_field_to_io(multi_dict: FileField) -> BytesIO: ... def file_field_to_io(file_field): @@ -77,7 +73,7 @@ def get_path_page(request: web.Request) -> Optional[int]: raise web.HTTPBadRequest(text=f"Invalid page value in path: {page_str}") if page < 1: - raise web.HTTPUnprocessableEntity(text=f"Page number must be greater than 1.") + raise web.HTTPUnprocessableEntity(text="Page number must be greater than 1.") return page diff --git a/src/aleph/web/controllers/version.py b/src/aleph/web/controllers/version.py index 5b41c82c8..37a869144 100644 --- a/src/aleph/web/controllers/version.py +++ b/src/aleph/web/controllers/version.py @@ -1,4 +1,5 @@ from aiohttp import web + from aleph import __version__ diff --git a/tests/api/conftest.py b/tests/api/conftest.py index 783304483..b45680b9a 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -1,20 +1,21 @@ import datetime as dt import json from pathlib import Path -from typing import Any, Dict, Sequence, cast, Tuple +from typing import Any, Dict, Sequence, Tuple, cast import pytest import pytest_asyncio from aleph_message.models import AggregateContent, PostContent from configmanager import Config +from in_memory_storage_engine import InMemoryStorageEngine from sqlalchemy import insert from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.accessors.aggregates import refresh_aggregate from aleph.db.models import ( - MessageDb, - ChainTxDb, AggregateElementDb, + ChainTxDb, + MessageDb, message_confirmations, ) from aleph.db.models.posts import PostDb @@ -23,7 +24,6 @@ from aleph.storage import StorageService from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.db_session import DbSessionFactory -from in_memory_storage_engine import InMemoryStorageEngine # TODO: remove the raw parameter, it's just to avoid larger refactorings diff --git a/tests/api/test_aggregates.py b/tests/api/test_aggregates.py index 06ee51a4f..9fdfc589b 100644 --- a/tests/api/test_aggregates.py +++ b/tests/api/test_aggregates.py @@ -2,6 +2,7 @@ import aiohttp import pytest + from aleph.db.models import MessageDb AGGREGATES_URI = "/api/v0/aggregates/{address}.json" @@ -166,6 +167,7 @@ async def test_get_aggregates_invalid_params( assert len(errors) == 1 assert errors[0]["loc"] == ["limit"], errors + @pytest.mark.asyncio async def test_get_aggregates_return_value_only( ccn_api_client, fixture_aggregate_messages: Sequence[MessageDb] @@ -178,8 +180,7 @@ async def test_get_aggregates_return_value_only( address, key = ADDRESS_1, "test_target" aggregates = await get_aggregates_expect_success( - ccn_api_client, address=address, keys=key, - with_info=False, value_only="1" + ccn_api_client, address=address, keys=key, with_info=False, value_only="1" ) assert address not in aggregates assert aggregates == EXPECTED_AGGREGATES[address][key] diff --git a/tests/api/test_balance.py b/tests/api/test_balance.py index ed30a97c4..c37d1b44c 100644 --- a/tests/api/test_balance.py +++ b/tests/api/test_balance.py @@ -1,9 +1,6 @@ -import json - import pytest -from aleph.db.models import ( - AlephBalanceDb, -) + +from aleph.db.models import AlephBalanceDb from aleph.jobs.process_pending_messages import PendingMessageProcessor MESSAGES_URI = "/api/v0/addresses/0x9319Ad3B7A8E0eE24f2E639c40D8eD124C5520Ba/balance" diff --git a/tests/api/test_get_message.py b/tests/api/test_get_message.py index c6df7b760..f2e13d82e 100644 --- a/tests/api/test_get_message.py +++ b/tests/api/test_get_message.py @@ -1,27 +1,27 @@ import datetime as dt -from typing import Mapping, Sequence, Any +from typing import Any, Mapping, Sequence import pytest import pytz from aleph_message.models import Chain, ItemType, MessageType from aleph.db.models import ( - MessageDb, ForgottenMessageDb, - RejectedMessageDb, + MessageDb, MessageStatusDb, PendingMessageDb, + RejectedMessageDb, ) from aleph.schemas.api.messages import ( - ProcessedMessageStatus, - RejectedMessageStatus, ForgottenMessageStatus, PendingMessageStatus, + ProcessedMessageStatus, + RejectedMessageStatus, ) from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.channel import Channel from aleph.types.db_session import DbSessionFactory -from aleph.types.message_status import MessageStatus, ErrorCode +from aleph.types.message_status import ErrorCode, MessageStatus MESSAGE_URI = "/api/v0/messages/{}" diff --git a/tests/api/test_list_messages.py b/tests/api/test_list_messages.py index b62a23958..93f2c1bae 100644 --- a/tests/api/test_list_messages.py +++ b/tests/api/test_list_messages.py @@ -5,10 +5,6 @@ import aiohttp import pytest -from aleph.db.models import MessageDb, PostDb -from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.channel import Channel -from aleph.types.db_session import DbSessionFactory from aleph_message.models import Chain, InstanceContent, ItemType, MessageType from aleph_message.models.execution.environment import ( FunctionEnvironment, @@ -17,6 +13,11 @@ from aleph_message.models.execution.instance import RootfsVolume from aleph_message.models.execution.volume import ImmutableVolume, ParentVolume +from aleph.db.models import MessageDb, PostDb +from aleph.toolkit.timestamp import timestamp_to_datetime +from aleph.types.channel import Channel +from aleph.types.db_session import DbSessionFactory + from .utils import get_messages_by_keys MESSAGES_URI = "/api/v0/messages.json" @@ -120,7 +121,7 @@ async def test_get_messages_filter_by_chain(fixture_messages, ccn_api_client): async def test_get_messages_filter_invalid_chain(fixture_messages, ccn_api_client): response = await fetch_messages_by_chain(api_client=ccn_api_client, chain="2CHAINZ") text = await response.text() - assert response.status == 422, await response.text() + assert response.status == 422, text async def fetch_messages_by_content_hash( diff --git a/tests/api/test_new_metric.py b/tests/api/test_new_metric.py index 2d8cdc9b5..5d25ca2fc 100644 --- a/tests/api/test_new_metric.py +++ b/tests/api/test_new_metric.py @@ -120,10 +120,8 @@ async def test_node_compute_metric(fixture_metrics_messages, ccn_api_client): @pytest.mark.asyncio async def test_node_compute_metric_not_exist(fixture_metrics_messages, ccn_api_client): - uri = _generate_uri( - "compute", "This_is_a_node_id" - ) + uri = _generate_uri("compute", "This_is_a_node_id") with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri) - assert response.status == 404 \ No newline at end of file + assert response.status == 404 diff --git a/tests/api/test_posts.py b/tests/api/test_posts.py index 72bc8f5a2..bd0e7b440 100644 --- a/tests/api/test_posts.py +++ b/tests/api/test_posts.py @@ -209,7 +209,7 @@ async def test_get_posts_tags( # Search for several tags response = await ccn_api_client.get( - "/api/v0/posts.json", params={"tags": f"mainnet,not-a-ref"} + "/api/v0/posts.json", params={"tags": "mainnet,not-a-ref"} ) assert response.status == 200 response_json = await response.json() @@ -225,7 +225,7 @@ async def test_get_posts_tags( # Check for several matching tags # Search for several tags response = await ccn_api_client.get( - "/api/v0/posts.json", params={"tags": f"original,mainnet"} + "/api/v0/posts.json", params={"tags": "original,mainnet"} ) assert response.status == 200 response_json = await response.json() diff --git a/tests/api/test_storage.py b/tests/api/test_storage.py index 7fc383dd3..5ceeab6f0 100644 --- a/tests/api/test_storage.py +++ b/tests/api/test_storage.py @@ -7,7 +7,7 @@ import orjson import pytest import pytest_asyncio -from aleph_message.models import ItemHash, Chain +from aleph_message.models import Chain, ItemHash from in_memory_storage_engine import InMemoryStorageEngine from aleph.chains.signature_verifier import SignatureVerifier @@ -17,7 +17,10 @@ from aleph.types.db_session import DbSessionFactory from aleph.types.files import FileType from aleph.types.message_status import MessageStatus -from aleph.web.controllers.app_state_getters import APP_STATE_SIGNATURE_VERIFIER, APP_STATE_STORAGE_SERVICE +from aleph.web.controllers.app_state_getters import ( + APP_STATE_SIGNATURE_VERIFIER, + APP_STATE_STORAGE_SERVICE, +) from aleph.web.controllers.utils import BroadcastStatus, PublicationStatus IPFS_ADD_FILE_URI = "/api/v0/ipfs/add_file" @@ -100,9 +103,7 @@ async def add_file_raw_upload( expected_file_hash: str, ): # Send the file content as raw bytes in the request body - headers = { - 'Content-Type': 'application/octet-stream' - } + headers = {"Content-Type": "application/octet-stream"} post_response = await api_client.post(uri, data=file_content, headers=headers) response_text = await post_response.text() assert post_response.status == 200, response_text @@ -255,8 +256,11 @@ async def test_storage_add_file(api_client, session_factory: DbSessionFactory): expected_file_hash=EXPECTED_FILE_SHA256, ) + @pytest.mark.asyncio -async def test_storage_add_file_raw_upload(api_client, session_factory: DbSessionFactory): +async def test_storage_add_file_raw_upload( + api_client, session_factory: DbSessionFactory +): await add_file_raw_upload( api_client, session_factory, @@ -416,4 +420,3 @@ async def test_ipfs_add_json(api_client, session_factory: DbSessionFactory): # creating a second fixture. expected_file_hash=ItemHash(EXPECTED_FILE_CID), ) - diff --git a/tests/api/utils/__init__.py b/tests/api/utils/__init__.py index 1616c798b..052534e8c 100644 --- a/tests/api/utils/__init__.py +++ b/tests/api/utils/__init__.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterable, List, Callable +from typing import Dict, Iterable, List def get_messages_by_keys(messages: Iterable[Dict], **keys) -> List[Dict]: diff --git a/tests/balances/test_balances.py b/tests/balances/test_balances.py index 2c355a484..9ccdd9505 100644 --- a/tests/balances/test_balances.py +++ b/tests/balances/test_balances.py @@ -1,4 +1,5 @@ -from typing import Optional, Mapping, Any +from decimal import Decimal +from typing import Any, Mapping, Optional import pytest from aleph_message.models import Chain @@ -6,8 +7,8 @@ from aleph.db.accessors.balances import get_balance_by_chain, get_total_balance from aleph.db.models import AlephBalanceDb from aleph.handlers.content.post import update_balances -from aleph.types.db_session import DbSessionFactory, DbSession -from decimal import Decimal +from aleph.types.db_session import DbSession, DbSessionFactory + BALANCES_CONTENT_SOL: Mapping[str, Any] = { "tags": ["SOL", "SPL", "CsZ5LZkDS7h9TDKjrbL7VAwQZ9nsRu8vJLhRYfmGaN8K", "mainnet"], "chain": "SOL", diff --git a/tests/chains/test_aleph_indexer.py b/tests/chains/test_aleph_indexer.py index 9bfeb1296..c13badf48 100644 --- a/tests/chains/test_aleph_indexer.py +++ b/tests/chains/test_aleph_indexer.py @@ -5,7 +5,7 @@ from configmanager import Config from aleph.chains.indexer_reader import AlephIndexerClient -from aleph.schemas.chains.indexer_response import IndexerBlockchain, EntityType +from aleph.schemas.chains.indexer_response import EntityType, IndexerBlockchain from aleph.types.chain_sync import ChainEventType diff --git a/tests/chains/test_avalanche.py b/tests/chains/test_avalanche.py index cc6c4a557..14dfde7cf 100644 --- a/tests/chains/test_avalanche.py +++ b/tests/chains/test_avalanche.py @@ -1,7 +1,7 @@ import pytest from aleph.chains.avalanche import AvalancheConnector -from aleph.schemas.pending_messages import parse_message, BasePendingMessage +from aleph.schemas.pending_messages import BasePendingMessage, parse_message @pytest.fixture diff --git a/tests/chains/test_chain_data_service.py b/tests/chains/test_chain_data_service.py index dad512ac6..cd2554f2d 100644 --- a/tests/chains/test_chain_data_service.py +++ b/tests/chains/test_chain_data_service.py @@ -1,13 +1,13 @@ -from typing import IO +import datetime as dt import pytest from aleph_message.models import ( Chain, - StoreContent, - MessageType, + ItemHash, ItemType, + MessageType, PostContent, - ItemHash, + StoreContent, ) from aleph.chains.chain_data_service import ChainDataService @@ -17,8 +17,7 @@ from aleph.schemas.pending_messages import parse_message from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.chain_sync import ChainSyncProtocol -from aleph.types.db_session import DbSessionFactory, DbSession -import datetime as dt +from aleph.types.db_session import DbSession, DbSessionFactory @pytest.mark.asyncio diff --git a/tests/chains/test_common.py b/tests/chains/test_common.py index 85f7b5298..e8280fb41 100644 --- a/tests/chains/test_common.py +++ b/tests/chains/test_common.py @@ -1,11 +1,12 @@ from unittest.mock import MagicMock import pytest -from aleph_message.models import MessageType, ItemType +from aleph_message.models import ItemType, MessageType -from aleph.chains.connector import ChainConnector from aleph.chains.common import get_verification_buffer +from aleph.chains.connector import ChainConnector from aleph.db.models import PendingMessageDb +from aleph.handlers.message_handler import MessageHandler from aleph.schemas.pending_messages import BasePendingMessage, parse_message from aleph.types.message_status import MessageProcessingStatus @@ -41,7 +42,7 @@ async def async_magic(): MagicMock.__await__ = lambda x: async_magic().__await__() - message_processor =MessageHandler(chain_service=ChainConnector()) + message_processor = MessageHandler(chain_service=ChainConnector()) mocker.patch("aleph.model.db") @@ -59,5 +60,7 @@ async def async_magic(): message = parse_message(message_dict) pending_message = PendingMessageDb.from_obj(message) - status, ops = await message_processor.verify_and_fetch_pending_message(pending_message) + status, ops = await message_processor.verify_and_fetch_pending_message( + pending_message + ) assert status == MessageProcessingStatus.MESSAGE_HANDLED diff --git a/tests/chains/test_confirmation.py b/tests/chains/test_confirmation.py index b0d411a0d..c874e3e81 100644 --- a/tests/chains/test_confirmation.py +++ b/tests/chains/test_confirmation.py @@ -79,16 +79,16 @@ async def test_confirm_message( ) pending_message = PendingMessageDb.from_message_dict( - MESSAGE_DICT, reception_time=dt.datetime(2022, 1, 1), fetched=True, + MESSAGE_DICT, + reception_time=dt.datetime(2022, 1, 1), + fetched=True, ) with session_factory() as session: await message_handler.process(session=session, pending_message=pending_message) session.commit() with session_factory() as session: - message_in_db = get_message_by_item_hash( - session=session, item_hash=item_hash - ) + message_in_db = get_message_by_item_hash(session=session, item_hash=item_hash) assert message_in_db is not None assert message_in_db.content == content @@ -103,15 +103,11 @@ async def test_confirm_message( pending_message.tx = chain_tx pending_message.tx_hash = chain_tx.hash - await message_handler.process( - session=session, pending_message=pending_message - ) + await message_handler.process(session=session, pending_message=pending_message) session.commit() with session_factory() as session: - message_in_db = get_message_by_item_hash( - session=session, item_hash=item_hash - ) + message_in_db = get_message_by_item_hash(session=session, item_hash=item_hash) assert message_in_db is not None assert message_in_db.confirmed @@ -147,19 +143,17 @@ async def test_process_confirmed_message( session.add(chain_tx) pending_message = PendingMessageDb.from_message_dict( - MESSAGE_DICT, reception_time=dt.datetime(2022, 1, 1), fetched=True, + MESSAGE_DICT, + reception_time=dt.datetime(2022, 1, 1), + fetched=True, ) pending_message.tx_hash = chain_tx.hash pending_message.tx = chain_tx - await message_handler.process( - session=session, pending_message=pending_message - ) + await message_handler.process(session=session, pending_message=pending_message) session.commit() with session_factory() as session: - message_in_db = get_message_by_item_hash( - session=session, item_hash=item_hash - ) + message_in_db = get_message_by_item_hash(session=session, item_hash=item_hash) assert message_in_db is not None assert message_in_db.confirmed diff --git a/tests/chains/test_cosmos.py b/tests/chains/test_cosmos.py index 395e9ba7c..7e61800b7 100644 --- a/tests/chains/test_cosmos.py +++ b/tests/chains/test_cosmos.py @@ -1,6 +1,7 @@ -import pytest import json +import pytest + from aleph.chains.cosmos import CosmosConnector TEST_MESSAGE = '{"chain": "CSDK", "channel": "TEST", "sender": "cosmos1rq3rcux05yftlh307gw8khh6xj43nv40mq27f5", "type": "AGGREGATE", "time": 1601997899.1849918, "item_content": "{\\"key\\":\\"test\\",\\"address\\":\\"cosmos1rq3rcux05yftlh307gw8khh6xj43nv40mq27f5\\",\\"content\\":{\\"a\\":1},\\"time\\":1601997899.1841497}", "item_hash": "248863f4eae7e31dfa33dd323af641550237c4ce90160aa1d96dd821a1b73221", "signature": "{\\"signature\\": \\"bbbdzrijw3i4eYoWhv6UI8Yqye480LOkJuNwkEp3S0sEshS9L+tQlbJSnfLDBwkC4VDh81uwhgmt57PEZws2cw==\\", \\"pub_key\\": {\\"type\\": \\"tendermint/PubKeySecp256k1\\", \\"value\\": \\"Ao3Ur9TXc/FktDebp9SnNCZWQaki/dq5G4GbiJH4aiu+\\"}, \\"account_number\\": \\"0\\", \\"sequence\\": \\"0\\"}", "content": {"key": "test", "address": "cosmos1rq3rcux05yftlh307gw8khh6xj43nv40mq27f5", "content": {"a": 1}, "time": 1601997899.1841497}}' diff --git a/tests/chains/test_nuls2.py b/tests/chains/test_nuls2.py index 26e12120a..bc3dba5d4 100644 --- a/tests/chains/test_nuls2.py +++ b/tests/chains/test_nuls2.py @@ -1,7 +1,8 @@ -from aleph.chains.nuls2 import Nuls2Connector, Nuls2Verifier -from aleph.schemas.pending_messages import parse_message import pytest +from aleph.chains.nuls2 import Nuls2Verifier +from aleph.schemas.pending_messages import parse_message + @pytest.mark.asyncio async def test_verify_signature_nuls2(mocker): diff --git a/tests/chains/test_solana.py b/tests/chains/test_solana.py index 80f4edae5..f4fc84862 100644 --- a/tests/chains/test_solana.py +++ b/tests/chains/test_solana.py @@ -1,5 +1,5 @@ import pytest -from aleph_message.models import MessageType, Chain, ItemType +from aleph_message.models import Chain, ItemType, MessageType from aleph.chains.solana import SolanaConnector from aleph.schemas.pending_messages import PendingPostMessage @@ -33,7 +33,7 @@ async def test_solana_signature(solana_message: PendingPostMessage): '{"signature": "bad-signature","publicKey": "AzfsDdCQp8uqzR4ProJ7yyLGKNp9iXHt92rEoiAHG4Pw"}', '{"signature": "56hzHaJHH3bz1DtU6Wjhyn7eLsMCYSY4HcXkKrpwAdEHFdw7k95NPzubR3x7otstbr5JCffw81Qqpd9YUq8XJuFj","publicKey": "AzfsDdCQp8uqzR4ProJ7yyLGKNp9iXHt92rEoiAHG4Pw"}', '{"signature": "56hzHaJHH3bz1DtU6Wjhyn7eLsMCYSY4HcXkKrpwAdEHFdw7k95NPzubR3x7otstbr5JCffw81Qqpd9YUq8XJuFj","publicKey": "BzfsDdCQp8uqzR4ProJ7yyLGKNp9iXHt92rEoiAHG4Pw"}', - '{{{{{', + "{{{{{", "56hzHaJHH3bz1DtU6Wjhyn7eLsMCYSY4HcXkKrpwAdEHFdw7k95NPzubR3x7otstbr5JCffw81Qqpd9YUq8XJuFj", ), ids=( diff --git a/tests/chains/test_substrate.py b/tests/chains/test_substrate.py index 08b971d43..72cae4d2f 100644 --- a/tests/chains/test_substrate.py +++ b/tests/chains/test_substrate.py @@ -9,7 +9,6 @@ TEST_MESSAGE_SIGNED_BY_PROVIDER = '{"chain": "DOT", "channel": "TEST", "sender": "5D9eKrAsfitxW48YrqMUXejcrnB2N8tLxPeFmoEZ4G74JFyz", "type": "POST", "item_type": "inline", "time": 1670865119.443, "item_content": "{\\"type\\":\\"Toolshed\\",\\"address\\":\\"5D9eKrAsfitxW48YrqMUXejcrnB2N8tLxPeFmoEZ4G74JFyz\\",\\"content\\":\\"Did the quick brown fox jump over the lazy dog?!\\",\\"time\\":1670865119.443}", "item_hash": "2f6e60df7ce1cdb2fb3be8ec09ffd20b5b781338984ff5f6f33830943f4397ba", "signature": "{\\"curve\\": \\"sr25519\\", \\"data\\": \\"0x866806b04e4cd99cd3a0f80232b8255d13c2782056a5e755fba5f233ccf8bf03a5fc2d708f8f70258b62d8e327da3a3ae6a280f1cab27eb912f125e0a1ade98a\\"}", "content": {"address": "5D9eKrAsfitxW48YrqMUXejcrnB2N8tLxPeFmoEZ4G74JFyz", "time": 1670865119.443, "content": "Did the quick brown fox jump over the lazy dog?!", "type": "Toolshed"}}' - @pytest.mark.asyncio async def test_verify_signature_real(): message_dict = json.loads(TEST_MESSAGE) diff --git a/tests/chains/test_tezos.py b/tests/chains/test_tezos.py index a87865339..aa0d93e4c 100644 --- a/tests/chains/test_tezos.py +++ b/tests/chains/test_tezos.py @@ -1,18 +1,12 @@ import datetime as dt import pytest -from aleph_message.models import ( - Chain, - MessageType, - AggregateContent, - PostContent, -) +from aleph_message.models import AggregateContent, Chain, MessageType, PostContent from aleph.chains.tezos import ( - TezosConnector, + TezosVerifier, datetime_to_iso_8601, indexer_event_to_chain_tx, - TezosVerifier, ) from aleph.db.models import PendingMessageDb from aleph.schemas.chains.tezos_indexer_response import ( diff --git a/tests/conftest.py b/tests/conftest.py index 2c14ba78a..2a9370c96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import sys from decimal import Decimal from pathlib import Path -from typing import Protocol, List +from typing import List, Protocol import alembic.command import alembic.config @@ -16,40 +16,39 @@ import pytest_asyncio import pytz from aleph_message.models import ( - MessageType, Chain, - ItemType, ExecutableContent, - ProgramContent, InstanceContent, + ItemType, + MessageType, + ProgramContent, ) from aleph_message.models.execution.volume import ImmutableVolume from configmanager import Config import aleph.config from aleph.db.accessors.files import insert_message_file_pin, upsert_file_tag -from aleph.db.connection import make_engine, make_session_factory, make_db_url +from aleph.db.connection import make_db_url, make_engine, make_session_factory from aleph.db.models import ( - PendingMessageDb, + AlephBalanceDb, MessageStatusDb, + PendingMessageDb, StoredFileDb, - AlephBalanceDb, ) from aleph.services.cache.node_cache import NodeCache from aleph.services.ipfs import IpfsService -from aleph.services.ipfs.common import make_ipfs_client from aleph.services.storage.fileystem_engine import FileSystemStorageEngine from aleph.storage import StorageService from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession -from aleph.types.files import FileType, FileTag +from aleph.types.db_session import DbSession, DbSessionFactory +from aleph.types.files import FileTag, FileType from aleph.types.message_status import MessageStatus from aleph.web import create_aiohttp_app from aleph.web.controllers.app_state_getters import ( APP_STATE_CONFIG, APP_STATE_P2P_CLIENT, - APP_STATE_STORAGE_SERVICE, APP_STATE_SESSION_FACTORY, + APP_STATE_STORAGE_SERVICE, ) # Add the helpers to the PYTHONPATH. diff --git a/tests/db/test_aggregates.py b/tests/db/test_aggregates.py index f17d6ac29..582742864 100644 --- a/tests/db/test_aggregates.py +++ b/tests/db/test_aggregates.py @@ -1,17 +1,17 @@ -from typing import Sequence, Tuple, Optional +import datetime as dt +from typing import Optional, Sequence, Tuple import pytest +import pytz import sqlalchemy.orm.exc from aleph.db.accessors.aggregates import ( get_aggregate_by_key, - refresh_aggregate, get_aggregate_content_keys, + refresh_aggregate, ) from aleph.db.models import AggregateDb, AggregateElementDb from aleph.types.db_session import DbSessionFactory -import datetime as dt -import pytz @pytest.mark.asyncio @@ -61,9 +61,7 @@ async def test_get_aggregate_by_key(session_factory: DbSessionFactory): @pytest.mark.asyncio async def test_get_aggregate_by_key_no_data(session_factory: DbSessionFactory): with session_factory() as session: - aggregate = get_aggregate_by_key( - session=session, owner="owner", key="key" - ) + aggregate = get_aggregate_by_key(session=session, owner="owner", key="key") assert aggregate is None @@ -214,8 +212,6 @@ async def test_get_content_keys( # Test no match with session_factory() as session: keys = set( - get_aggregate_content_keys( - session=session, key="not-a-key", owner="no-one" - ) + get_aggregate_content_keys(session=session, key="not-a-key", owner="no-one") ) assert keys == set() diff --git a/tests/db/test_chains.py b/tests/db/test_chains.py index 8ed53ae67..c4e2004aa 100644 --- a/tests/db/test_chains.py +++ b/tests/db/test_chains.py @@ -5,19 +5,18 @@ from aleph_message.models import Chain from sqlalchemy import select -from aleph.toolkit.range import MultiRange, Range -from aleph.types.chain_sync import ChainEventType -from aleph.types.db_session import DbSessionFactory - from aleph.db.accessors.chains import ( - upsert_chain_sync_status, - get_last_height, IndexerMultiRange, - update_indexer_multirange, get_indexer_multirange, + get_last_height, get_missing_indexer_datetime_multirange, + update_indexer_multirange, + upsert_chain_sync_status, ) from aleph.db.models.chains import ChainSyncStatusDb, IndexerSyncStatusDb +from aleph.toolkit.range import MultiRange, Range +from aleph.types.chain_sync import ChainEventType +from aleph.types.db_session import DbSessionFactory @pytest.mark.asyncio diff --git a/tests/db/test_cost.py b/tests/db/test_cost.py index 9ab90c12e..3435d13f2 100644 --- a/tests/db/test_cost.py +++ b/tests/db/test_cost.py @@ -1,33 +1,31 @@ +import datetime as dt +import json +from decimal import Decimal from typing import List, Protocol import pytest import pytz from aleph_message.models import ( Chain, + ExecutableContent, InstanceContent, - MessageType, ItemType, - ExecutableContent, + MessageType, ProgramContent, ) -from decimal import Decimal - from aleph_message.models.execution.volume import ImmutableVolume from aleph.db.accessors.cost import get_total_cost_for_address from aleph.db.accessors.files import insert_message_file_pin, upsert_file_tag from aleph.db.models import ( AlephBalanceDb, + MessageStatusDb, PendingMessageDb, StoredFileDb, - MessageStatusDb, ) -import json from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession -import datetime as dt - -from aleph.types.files import FileType, FileTag +from aleph.types.db_session import DbSession, DbSessionFactory +from aleph.types.files import FileTag, FileType from aleph.types.message_status import MessageStatus @@ -207,4 +205,6 @@ def test_get_total_cost_for_address( total_cost: Decimal = get_total_cost_for_address( session=session, address=fixture_instance_message.sender ) - assert total_cost == Decimal(0.66666666666666662965923251249478198587894439697265625) + assert total_cost == Decimal( + 0.66666666666666662965923251249478198587894439697265625 + ) diff --git a/tests/db/test_files.py b/tests/db/test_files.py index 4b2592c5c..7927fa599 100644 --- a/tests/db/test_files.py +++ b/tests/db/test_files.py @@ -5,14 +5,14 @@ import pytz from aleph.db.accessors.files import ( - is_pinned_file, - upsert_file_tag, get_file_tag, + is_pinned_file, refresh_file_tag, + upsert_file_tag, ) -from aleph.db.models import TxFilePinDb, StoredFileDb, MessageFilePinDb +from aleph.db.models import MessageFilePinDb, StoredFileDb, TxFilePinDb from aleph.types.db_session import DbSessionFactory -from aleph.types.files import FileType, FileTag +from aleph.types.files import FileTag, FileType @pytest.mark.asyncio diff --git a/tests/db/test_messages.py b/tests/db/test_messages.py index 123584303..b8b211a0f 100644 --- a/tests/db/test_messages.py +++ b/tests/db/test_messages.py @@ -3,27 +3,22 @@ import pytest import pytz -from aleph_message.models import Chain, MessageType, ItemType -from sqlalchemy import select, insert, text +from aleph_message.models import Chain, ItemType, MessageType +from sqlalchemy import insert, select, text from aleph.db.accessors.messages import ( - get_message_by_item_hash, - get_unconfirmed_messages, - message_exists, - forget_message, - get_message_status, append_to_forgotten_by, + forget_message, + get_distinct_channels, get_forgotten_message, - make_message_upsert_query, + get_message_by_item_hash, + get_message_status, + get_unconfirmed_messages, make_confirmation_upsert_query, - get_distinct_channels, -) -from aleph.db.models import ( - MessageDb, - ChainTxDb, - MessageStatusDb, - message_confirmations, + make_message_upsert_query, + message_exists, ) +from aleph.db.models import ChainTxDb, MessageDb, MessageStatusDb, message_confirmations from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.chain_sync import ChainSyncProtocol from aleph.types.channel import Channel @@ -148,20 +143,18 @@ async def test_get_message_with_confirmations( @pytest.mark.asyncio async def test_message_exists(session_factory: DbSessionFactory, fixture_message): with session_factory() as session: - assert not message_exists( - session=session, item_hash=fixture_message.item_hash - ) + assert not message_exists(session=session, item_hash=fixture_message.item_hash) session.add(fixture_message) session.commit() - assert message_exists( - session=session, item_hash=fixture_message.item_hash - ) + assert message_exists(session=session, item_hash=fixture_message.item_hash) @pytest.mark.asyncio -async def test_message_count(session_factory: DbSessionFactory, fixture_message: MessageDb): +async def test_message_count( + session_factory: DbSessionFactory, fixture_message: MessageDb +): with session_factory() as session: session.add(fixture_message) session.commit() @@ -299,9 +292,7 @@ async def test_get_unconfirmed_messages( assert unconfirmed_messages == [] # Check that it is also ignored when the chain parameter is specified - unconfirmed_messages = list( - get_unconfirmed_messages(session, chain=tx.chain) - ) + unconfirmed_messages = list(get_unconfirmed_messages(session, chain=tx.chain)) assert unconfirmed_messages == [] # Check that it reappears if we specify a different chain @@ -319,7 +310,9 @@ async def test_get_unconfirmed_messages( @pytest.mark.asyncio -async def test_get_unconfirmed_messages_trusted_messages(session_factory:DbSessionFactory, fixture_message: MessageDb): +async def test_get_unconfirmed_messages_trusted_messages( + session_factory: DbSessionFactory, fixture_message: MessageDb +): fixture_message.signature = None with session_factory() as session: session.add(fixture_message) diff --git a/tests/db/test_peers.py b/tests/db/test_peers.py index de85ebc44..3826b3997 100644 --- a/tests/db/test_peers.py +++ b/tests/db/test_peers.py @@ -4,7 +4,7 @@ import pytz from sqlalchemy import select -from aleph.db.accessors.peers import upsert_peer, get_all_addresses_by_peer_type +from aleph.db.accessors.peers import get_all_addresses_by_peer_type, upsert_peer from aleph.db.models.peers import PeerDb, PeerType from aleph.types.db_session import DbSessionFactory diff --git a/tests/db/test_pending_messages_db.py b/tests/db/test_pending_messages_db.py index f43236cdf..ea1c46ff1 100644 --- a/tests/db/test_pending_messages_db.py +++ b/tests/db/test_pending_messages_db.py @@ -1,16 +1,16 @@ +import datetime as dt from typing import List import pytest -from aleph_message.models import ItemType, Chain, MessageType +from aleph_message.models import Chain, ItemType, MessageType from aleph.db.accessors.pending_messages import ( count_pending_messages, get_next_pending_messages, ) -from aleph.db.models import PendingMessageDb, ChainTxDb +from aleph.db.models import ChainTxDb, PendingMessageDb from aleph.types.chain_sync import ChainSyncProtocol from aleph.types.db_session import DbSessionFactory -import datetime as dt @pytest.fixture diff --git a/tests/db/test_pending_txs.py b/tests/db/test_pending_txs.py index 49954d82a..caf57f8f8 100644 --- a/tests/db/test_pending_txs.py +++ b/tests/db/test_pending_txs.py @@ -1,13 +1,12 @@ +import datetime as dt from typing import Sequence import pytest from aleph_message.models import Chain -from aleph.db.accessors.pending_txs import get_pending_txs, count_pending_txs +from aleph.db.accessors.pending_txs import count_pending_txs, get_pending_txs from aleph.db.models import ChainTxDb, PendingTxDb from aleph.types.chain_sync import ChainSyncProtocol -import datetime as dt - from aleph.types.db_session import DbSessionFactory diff --git a/tests/db/test_posts.py b/tests/db/test_posts.py index 69d1c5fa2..e7185720e 100644 --- a/tests/db/test_posts.py +++ b/tests/db/test_posts.py @@ -1,28 +1,28 @@ import datetime as dt -from typing import Optional, Any, Dict +import json +from typing import Any, Dict, Optional import pytest import pytz -from aleph_message.models import ItemHash, ItemType, MessageType, Chain +from aleph_message.models import Chain, ItemHash, ItemType, MessageType from more_itertools import one from aleph.db.accessors.posts import ( - get_post, MergedPost, - get_matching_posts, + MergedPostV0, count_matching_posts, - refresh_latest_amend, - get_original_post, delete_post, + get_matching_posts, get_matching_posts_legacy, - MergedPostV0, + get_original_post, + get_post, + refresh_latest_amend, ) from aleph.db.models import MessageDb from aleph.db.models.posts import PostDb from aleph.types.channel import Channel from aleph.types.db_session import DbSessionFactory from aleph.types.sort_order import SortOrder -import json def message_fields_from_post(post: PostDb) -> Dict[str, Any]: @@ -63,7 +63,7 @@ def original_message(original_post: PostDb) -> MessageDb: return MessageDb( **message_fields_from_post(original_post), chain=Chain.ETH, - signature="sig-original" + signature="sig-original", ) @@ -88,7 +88,7 @@ def first_amend_message(first_amend_post: PostDb) -> MessageDb: return MessageDb( **message_fields_from_post(first_amend_post), chain=Chain.ETH, - signature="sig-first-amend" + signature="sig-first-amend", ) @@ -113,7 +113,7 @@ def second_amend_message(second_amend_post: PostDb) -> MessageDb: return MessageDb( **message_fields_from_post(second_amend_post), chain=Chain.ETH, - signature="sig-second-amend" + signature="sig-second-amend", ) @@ -140,7 +140,7 @@ def message_from_second_user(post_from_second_user: PostDb) -> MessageDb: return MessageDb( **message_fields_from_post(post_from_second_user), chain=Chain.ETH, - signature="sig-post-from-second-user" + signature="sig-post-from-second-user", ) diff --git a/tests/db/test_programs_db.py b/tests/db/test_programs_db.py index f48267b7c..eb03a347a 100644 --- a/tests/db/test_programs_db.py +++ b/tests/db/test_programs_db.py @@ -5,26 +5,26 @@ import pytest import pytz from aleph_message.models import ItemHash -from aleph_message.models.execution.program import MachineType, Encoding +from aleph_message.models.execution.program import Encoding, MachineType from aleph_message.models.execution.volume import VolumePersistence from sqlalchemy import select from aleph.db.accessors.vms import ( + delete_vm, get_program, is_vm_amend_allowed, refresh_vm_version, - delete_vm, ) from aleph.db.models import ( - VmBaseDb, CodeVolumeDb, - RuntimeDb, - VmVersionDb, DataVolumeDb, + EphemeralVolumeDb, ExportVolumeDb, ImmutableVolumeDb, - EphemeralVolumeDb, - PersistentVolumeDb, ProgramDb, + PersistentVolumeDb, + ProgramDb, + RuntimeDb, + VmVersionDb, ) from aleph.types.db_session import DbSessionFactory from aleph.types.vms import VmVersion diff --git a/tests/helpers/message_test_helpers.py b/tests/helpers/message_test_helpers.py index 0acdf4061..ef79e851a 100644 --- a/tests/helpers/message_test_helpers.py +++ b/tests/helpers/message_test_helpers.py @@ -1,11 +1,11 @@ import datetime as dt import itertools import json -from typing import List, Optional, Union, Any, Mapping, Sequence, Iterable +from typing import Any, Iterable, List, Mapping, Optional, Sequence, Union from aleph_message.models import ItemType, MessageConfirmation -from aleph.db.models import MessageDb, PendingMessageDb, MessageStatusDb +from aleph.db.models import MessageDb, MessageStatusDb, PendingMessageDb from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.toolkit.timestamp import utc_now from aleph.types.db_session import DbSession diff --git a/tests/message_processing/conftest.py b/tests/message_processing/conftest.py index da37e723d..eb08606b4 100644 --- a/tests/message_processing/conftest.py +++ b/tests/message_processing/conftest.py @@ -6,15 +6,15 @@ import pytest import pytest_asyncio from configmanager import Config +from in_memory_storage_engine import InMemoryStorageEngine -from aleph.chains.connector import ChainConnector from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.models import ChainTxDb, PendingMessageDb from aleph.handlers.message_handler import MessageHandler from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.storage import StorageService from aleph.types.db_session import DbSessionFactory -from in_memory_storage_engine import InMemoryStorageEngine + from .load_fixtures import load_fixture_messages diff --git a/tests/message_processing/test_process_aggregates.py b/tests/message_processing/test_process_aggregates.py index 8fd199761..68822cfca 100644 --- a/tests/message_processing/test_process_aggregates.py +++ b/tests/message_processing/test_process_aggregates.py @@ -1,27 +1,27 @@ import datetime as dt import json -from typing import Dict, List, Sequence, Iterable +from typing import Dict, Iterable, List, Sequence import pytest import pytz -from aleph_message.models import ItemType, Chain, MessageType, AggregateContent +from aleph_message.models import AggregateContent, Chain, ItemType, MessageType from configmanager import Config +from message_test_helpers import process_pending_messages from more_itertools import one from sqlalchemy import select from sqlalchemy.orm import selectinload from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.accessors.aggregates import get_aggregate_by_key, get_aggregate_elements -from aleph.db.models import PendingMessageDb, MessageDb, AggregateElementDb, AggregateDb +from aleph.db.models import AggregateDb, AggregateElementDb, MessageDb, PendingMessageDb from aleph.handlers.content.aggregate import AggregateMessageHandler from aleph.handlers.message_handler import MessageHandler from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.storage import StorageService from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.channel import Channel -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.message_processing_result import ProcessedMessage -from message_test_helpers import process_pending_messages @pytest.mark.asyncio @@ -96,7 +96,7 @@ async def test_process_aggregates( fixture_aggregate_messages: List[Dict], ): pipeline = message_processor.make_pipeline() - messages = [message async for message in pipeline] + [message async for message in pipeline] # TODO: improve this test diff --git a/tests/message_processing/test_process_forgets.py b/tests/message_processing/test_process_forgets.py index f706cbad9..5f10420cf 100644 --- a/tests/message_processing/test_process_forgets.py +++ b/tests/message_processing/test_process_forgets.py @@ -1,28 +1,29 @@ import datetime as dt import pytest -from aleph_message.models import Chain, MessageType, ItemType +from aleph_message.models import Chain, ItemType, MessageType from configmanager import Config +from message_test_helpers import process_pending_messages from more_itertools import one from sqlalchemy import select from aleph.db.accessors.files import count_file_pins, get_file -from aleph.db.accessors.messages import get_message_status, get_forgotten_message +from aleph.db.accessors.messages import get_forgotten_message, get_message_status from aleph.db.accessors.posts import get_post from aleph.db.models import ( - PendingMessageDb, - StoredFileDb, - MessageDb, - MessageStatusDb, FilePinDb, - MessageFilePinDb, GracePeriodFilePinDb, + MessageDb, + MessageFilePinDb, + MessageStatusDb, + PendingMessageDb, + StoredFileDb, ) from aleph.handlers.content.aggregate import AggregateMessageHandler from aleph.handlers.content.forget import ForgetMessageHandler from aleph.handlers.content.post import PostMessageHandler -from aleph.handlers.content.vm import VmMessageHandler from aleph.handlers.content.store import StoreMessageHandler +from aleph.handlers.content.vm import VmMessageHandler from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.channel import Channel @@ -30,9 +31,6 @@ from aleph.types.files import FileType from aleph.types.message_processing_result import ProcessedMessage, RejectedMessage from aleph.types.message_status import MessageStatus -from message_test_helpers import ( - process_pending_messages, -) @pytest.fixture diff --git a/tests/message_processing/test_process_instances.py b/tests/message_processing/test_process_instances.py index 6d38a83e6..78e776d47 100644 --- a/tests/message_processing/test_process_instances.py +++ b/tests/message_processing/test_process_instances.py @@ -1,38 +1,35 @@ import datetime as dt import itertools import json -from typing import List, Protocol from decimal import Decimal +from typing import List, Protocol import pytest import pytz from aleph_message.models import ( - ItemType, Chain, - MessageType, - InstanceContent, ExecutableContent, ForgetContent, + InstanceContent, + ItemType, + MessageType, ) from aleph_message.models.execution.program import ProgramContent from aleph_message.models.execution.volume import ImmutableVolume from more_itertools import one from sqlalchemy import text -from aleph.db.accessors.files import ( - insert_message_file_pin, - upsert_file_tag, -) +from aleph.db.accessors.files import insert_message_file_pin, upsert_file_tag from aleph.db.accessors.messages import get_message_status, get_rejected_message from aleph.db.accessors.vms import get_instance, get_vm_version from aleph.db.models import ( - PendingMessageDb, - MessageStatusDb, - ImmutableVolumeDb, + AlephBalanceDb, EphemeralVolumeDb, + ImmutableVolumeDb, + MessageStatusDb, + PendingMessageDb, PersistentVolumeDb, StoredFileDb, - AlephBalanceDb, ) from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.services.cost import ( @@ -41,9 +38,9 @@ get_volume_size, ) from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.files import FileTag, FileType -from aleph.types.message_status import MessageStatus, ErrorCode +from aleph.types.message_status import ErrorCode, MessageStatus @pytest.fixture @@ -323,7 +320,7 @@ async def test_process_instance( assert len(volumes_by_type[PersistentVolumeDb]) == 3 assert len(volumes_by_type[ImmutableVolumeDb]) == 1 - ephemeral_volume: EphemeralVolumeDb = one(volumes_by_type[EphemeralVolumeDb]) # type: ignore[assignment] + ephemeral_volume: EphemeralVolumeDb = one(volumes_by_type[EphemeralVolumeDb]) assert ephemeral_volume.mount == "/var/cache" assert ephemeral_volume.size_mib == 5 diff --git a/tests/message_processing/test_process_pending_txs.py b/tests/message_processing/test_process_pending_txs.py index a07580fd1..d9f3b6086 100644 --- a/tests/message_processing/test_process_pending_txs.py +++ b/tests/message_processing/test_process_pending_txs.py @@ -8,7 +8,7 @@ from sqlalchemy import select from aleph.chains.chain_data_service import ChainDataService -from aleph.db.models import PendingMessageDb, MessageStatusDb +from aleph.db.models import MessageStatusDb, PendingMessageDb from aleph.db.models.chains import ChainTxDb from aleph.db.models.pending_txs import PendingTxDb from aleph.handlers.message_handler import MessagePublisher @@ -19,6 +19,7 @@ from aleph.types.chain_sync import ChainSyncProtocol from aleph.types.db_session import DbSessionFactory from aleph.types.message_status import MessageStatus + from .load_fixtures import load_fixture_messages diff --git a/tests/message_processing/test_process_programs.py b/tests/message_processing/test_process_programs.py index ec0a1e7b1..2968d17fb 100644 --- a/tests/message_processing/test_process_programs.py +++ b/tests/message_processing/test_process_programs.py @@ -1,40 +1,36 @@ import datetime as dt import itertools import json +from decimal import Decimal from typing import List import pytest import pytz -from aleph_message.models import ItemType, Chain, MessageType -from aleph_message.models.execution.program import ( - MachineType, - ProgramContent, -) +from aleph_message.models import Chain, ItemType, MessageType +from aleph_message.models.execution.program import MachineType, ProgramContent from aleph_message.models.execution.volume import ImmutableVolume, VolumePersistence from more_itertools import one from sqlalchemy import select -from aleph.db.accessors.files import ( - insert_message_file_pin, - upsert_file_tag, -) +from aleph.db.accessors.files import insert_message_file_pin, upsert_file_tag from aleph.db.accessors.messages import get_message_status, get_rejected_message from aleph.db.accessors.vms import get_program from aleph.db.models import ( - PendingMessageDb, - MessageStatusDb, - VmBaseDb, - ImmutableVolumeDb, + AlephBalanceDb, EphemeralVolumeDb, + ImmutableVolumeDb, + MessageStatusDb, + PendingMessageDb, PersistentVolumeDb, - StoredFileDb, AlephBalanceDb, + StoredFileDb, + VmBaseDb, ) from aleph.jobs.process_pending_messages import PendingMessageProcessor from aleph.toolkit.timestamp import timestamp_to_datetime -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.files import FileTag, FileType -from aleph.types.message_status import MessageStatus, ErrorCode -from decimal import Decimal +from aleph.types.message_status import ErrorCode, MessageStatus + @pytest.fixture def fixture_program_message(session_factory: DbSessionFactory) -> PendingMessageDb: @@ -148,6 +144,7 @@ def insert_volume_refs(session: DbSession, message: PendingMessageDb): last_updated=created, ) + @pytest.fixture def user_balance(session_factory: DbSessionFactory) -> AlephBalanceDb: balance = AlephBalanceDb( @@ -162,6 +159,7 @@ def user_balance(session_factory: DbSessionFactory) -> AlephBalanceDb: session.commit() return balance + @pytest.mark.asyncio async def test_process_program( user_balance: AlephBalanceDb, @@ -228,7 +226,7 @@ async def test_process_program( assert len(volumes_by_type[PersistentVolumeDb]) == 1 assert len(volumes_by_type[ImmutableVolumeDb]) == 2 - persistent_volume: PersistentVolumeDb = one(volumes_by_type[PersistentVolumeDb]) # type: ignore[assignment] + persistent_volume: PersistentVolumeDb = one(volumes_by_type[PersistentVolumeDb]) assert persistent_volume.name == "data" assert persistent_volume.mount == "/data" assert persistent_volume.size_mib == 128 @@ -251,7 +249,7 @@ async def test_program_with_subscriptions( _ = [message async for message in pipeline] assert program_message.item_content - content_dict = json.loads(program_message.item_content) + json.loads(program_message.item_content) with session_factory() as session: program: VmBaseDb = session.execute(select(VmBaseDb)).scalar_one() diff --git a/tests/message_processing/test_process_stores.py b/tests/message_processing/test_process_stores.py index 90ba8829d..294b2816f 100644 --- a/tests/message_processing/test_process_stores.py +++ b/tests/message_processing/test_process_stores.py @@ -1,15 +1,14 @@ import datetime as dt import json -from typing import Optional, Mapping +from typing import Mapping, Optional import pytest -from aleph_message.models import Chain, MessageType, ItemType +from aleph_message.models import Chain, ItemType, MessageType from configmanager import Config -from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.accessors.files import get_message_file_pin from aleph.db.accessors.messages import get_message_by_item_hash -from aleph.db.models import PendingMessageDb, MessageStatusDb +from aleph.db.models import MessageStatusDb, PendingMessageDb from aleph.handlers.content.store import StoreMessageHandler from aleph.handlers.message_handler import MessageHandler from aleph.jobs.process_pending_messages import PendingMessageProcessor diff --git a/tests/permissions/test_check_sender_authorization.py b/tests/permissions/test_check_sender_authorization.py index aeb0b5c46..6da789607 100644 --- a/tests/permissions/test_check_sender_authorization.py +++ b/tests/permissions/test_check_sender_authorization.py @@ -2,12 +2,12 @@ from typing import Any, Mapping import pytest +from message_test_helpers import make_validated_message_from_dict from aleph.db.models import AggregateDb, AggregateElementDb from aleph.permissions import check_sender_authorization from aleph.toolkit.timestamp import timestamp_to_datetime from aleph.types.db_session import DbSessionFactory -from message_test_helpers import make_validated_message_from_dict @pytest.mark.asyncio diff --git a/tests/services/test_cost_service.py b/tests/services/test_cost_service.py index 151c8fbf3..5afb3eeb1 100644 --- a/tests/services/test_cost_service.py +++ b/tests/services/test_cost_service.py @@ -1,17 +1,16 @@ from decimal import Decimal +from unittest.mock import Mock import pytest -from aleph.types.db_session import DbSession - from aleph_message.models import ExecutableContent, InstanceContent from aleph.services.cost import ( + HOUR, compute_cost, compute_flow_cost, - get_additional_storage_price, HOUR, + get_additional_storage_price, ) - -from unittest.mock import Mock +from aleph.types.db_session import DbSession class StoredFileDb: @@ -25,31 +24,25 @@ def fixture_hold_instance_message() -> ExecutableContent: "rootfs": { "parent": { "ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595", - "use_latest": True + "use_latest": True, }, "size_mib": 20480, - "persistence": "host" + "persistence": "host", }, "address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", "volumes": [], - "metadata": { - "name": "Test Debian 12" - }, - "resources": { - "vcpus": 1, - "memory": 2048, - "seconds": 30 - }, + "metadata": {"name": "Test Debian 12"}, + "resources": {"vcpus": 1, "memory": 2048, "seconds": 30}, "allow_amend": False, "environment": { "internet": True, "aleph_api": True, "reproducible": False, - "shared_cache": False + "shared_cache": False, }, "authorized_keys": [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y nesitor@gmail.com" - ] + ], } return InstanceContent.parse_obj(content) @@ -62,10 +55,10 @@ def fixture_hold_instance_message_complete() -> ExecutableContent: "rootfs": { "parent": { "ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595", - "use_latest": True + "use_latest": True, }, "size_mib": 20480, - "persistence": "host" + "persistence": "host", }, "address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", "volumes": [ @@ -84,8 +77,8 @@ def fixture_hold_instance_message_complete() -> ExecutableContent: }, { "comment": "Working data persisted on the Aleph network. " - "New VMs will try to use the latest version of this volume, " - "with no guarantee against conflicts", + "New VMs will try to use the latest version of this volume, " + "with no guarantee against conflicts", "mount": "/var/lib/statistics", "name": "statistics", "persistence": "store", @@ -98,24 +91,18 @@ def fixture_hold_instance_message_complete() -> ExecutableContent: "size_mib": 100, }, ], - "metadata": { - "name": "Test Debian 12" - }, - "resources": { - "vcpus": 1, - "memory": 2048, - "seconds": 30 - }, + "metadata": {"name": "Test Debian 12"}, + "resources": {"vcpus": 1, "memory": 2048, "seconds": 30}, "allow_amend": False, "environment": { "internet": True, "aleph_api": True, "reproducible": False, - "shared_cache": False + "shared_cache": False, }, "authorized_keys": [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y nesitor@gmail.com" - ] + ], } return InstanceContent.parse_obj(content) @@ -128,36 +115,30 @@ def fixture_flow_instance_message() -> ExecutableContent: "rootfs": { "parent": { "ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595", - "use_latest": True + "use_latest": True, }, "size_mib": 20480, - "persistence": "host" + "persistence": "host", }, "address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", "volumes": [], - "metadata": { - "name": "Test Debian 12" - }, - "resources": { - "vcpus": 1, - "memory": 2048, - "seconds": 30 - }, + "metadata": {"name": "Test Debian 12"}, + "resources": {"vcpus": 1, "memory": 2048, "seconds": 30}, "allow_amend": False, "environment": { "internet": True, "aleph_api": True, "reproducible": False, - "shared_cache": False + "shared_cache": False, }, "payment": { "chain": "AVAX", "receiver": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", - "type": "superfluid" + "type": "superfluid", }, "authorized_keys": [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y nesitor@gmail.com" - ] + ], } return InstanceContent.parse_obj(content) @@ -170,10 +151,10 @@ def fixture_flow_instance_message_complete() -> ExecutableContent: "rootfs": { "parent": { "ref": "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595", - "use_latest": True + "use_latest": True, }, "size_mib": 20480, - "persistence": "host" + "persistence": "host", }, "address": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", "volumes": [ @@ -192,8 +173,8 @@ def fixture_flow_instance_message_complete() -> ExecutableContent: }, { "comment": "Working data persisted on the Aleph network. " - "New VMs will try to use the latest version of this volume, " - "with no guarantee against conflicts", + "New VMs will try to use the latest version of this volume, " + "with no guarantee against conflicts", "mount": "/var/lib/statistics", "name": "statistics", "persistence": "store", @@ -206,29 +187,23 @@ def fixture_flow_instance_message_complete() -> ExecutableContent: "size_mib": 51200, }, ], - "metadata": { - "name": "Test Debian 12" - }, - "resources": { - "vcpus": 1, - "memory": 2048, - "seconds": 30 - }, + "metadata": {"name": "Test Debian 12"}, + "resources": {"vcpus": 1, "memory": 2048, "seconds": 30}, "allow_amend": False, "environment": { "internet": True, "aleph_api": True, "reproducible": False, - "shared_cache": False + "shared_cache": False, }, "payment": { "chain": "AVAX", "receiver": "0xA07B1214bAe0D5ccAA25449C3149c0aC83658874", - "type": "superfluid" + "type": "superfluid", }, "authorized_keys": [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHlGJRaIv/EzNT0eNqNB5DiGEbii28Fb2zCjuO/bMu7y nesitor@gmail.com" - ] + ], } return InstanceContent.parse_obj(content) @@ -237,7 +212,7 @@ def fixture_flow_instance_message_complete() -> ExecutableContent: def test_compute_cost(fixture_hold_instance_message): file_db = StoredFileDb() mock = Mock() - mock.patch('_get_file_from_ref', return_value=file_db) + mock.patch("_get_file_from_ref", return_value=file_db) cost = compute_cost(content=fixture_hold_instance_message, session=DbSession()) assert cost == 2000 @@ -245,34 +220,40 @@ def test_compute_cost(fixture_hold_instance_message): def test_get_additional_storage_price(fixture_hold_instance_message): file_db = StoredFileDb() mock = Mock() - mock.patch('_get_file_from_ref', return_value=file_db) - cost = get_additional_storage_price(content=fixture_hold_instance_message, session=DbSession()) + mock.patch("_get_file_from_ref", return_value=file_db) + cost = get_additional_storage_price( + content=fixture_hold_instance_message, session=DbSession() + ) assert cost == 0 def test_compute_cost_complete(fixture_hold_instance_message_complete): file_db = StoredFileDb() mock = Mock() - mock.patch('_get_file_from_ref', return_value=file_db) - cost = compute_cost(content=fixture_hold_instance_message_complete, session=DbSession()) + mock.patch("_get_file_from_ref", return_value=file_db) + cost = compute_cost( + content=fixture_hold_instance_message_complete, session=DbSession() + ) assert cost == 2017.50 def test_compute_flow_cost(fixture_flow_instance_message): file_db = StoredFileDb() mock = Mock() - mock.patch('_get_file_from_ref', return_value=file_db) + mock.patch("_get_file_from_ref", return_value=file_db) cost = compute_flow_cost(content=fixture_flow_instance_message, session=DbSession()) - assert cost == Decimal('0.00003055555555555555555555555556') + assert cost == Decimal("0.00003055555555555555555555555556") cost_per_hour = cost * HOUR - assert cost_per_hour == Decimal('0.11') + assert cost_per_hour == Decimal("0.11") def test_compute_flow_cost_complete(fixture_flow_instance_message_complete): file_db = StoredFileDb() mock = Mock() - mock.patch('_get_file_from_ref', return_value=file_db) - cost = compute_flow_cost(content=fixture_flow_instance_message_complete, session=DbSession()) - assert cost == Decimal('0.00004752116055555555555555555556') + mock.patch("_get_file_from_ref", return_value=file_db) + cost = compute_flow_cost( + content=fixture_flow_instance_message_complete, session=DbSession() + ) + assert cost == Decimal("0.00004752116055555555555555555556") cost_per_hour = cost * HOUR - assert cost_per_hour == Decimal('0.171076178') + assert cost_per_hour == Decimal("0.171076178") diff --git a/tests/services/test_garbage_collector.py b/tests/services/test_garbage_collector.py index 2b77e6f07..cdde1d4a5 100644 --- a/tests/services/test_garbage_collector.py +++ b/tests/services/test_garbage_collector.py @@ -3,19 +3,18 @@ import pytest import pytest_asyncio -from sqlalchemy import select from aleph.db.accessors.files import get_file from aleph.db.models import ( - StoredFileDb, - MessageFilePinDb, GracePeriodFilePinDb, + MessageFilePinDb, + StoredFileDb, TxFilePinDb, ) from aleph.services.storage.engine import StorageEngine from aleph.services.storage.garbage_collector import GarbageCollector from aleph.storage import StorageService -from aleph.types.db_session import DbSessionFactory, DbSession +from aleph.types.db_session import DbSession, DbSessionFactory from aleph.types.files import FileType diff --git a/tests/services/test_utils.py b/tests/services/test_utils.py index 930e60151..1ec90c232 100644 --- a/tests/services/test_utils.py +++ b/tests/services/test_utils.py @@ -1,10 +1,10 @@ import pytest from aleph.services.utils import ( - is_valid_ip4, + get_IP, get_ip4_from_service, get_ip4_from_socket, - get_IP, + is_valid_ip4, ) diff --git a/tests/storage/test_get_content.py b/tests/storage/test_get_content.py index 05d124d87..920d26b02 100644 --- a/tests/storage/test_get_content.py +++ b/tests/storage/test_get_content.py @@ -1,14 +1,13 @@ import json -from typing import Optional, Dict +from typing import Dict, Optional import pytest -from aleph.services.ipfs import IpfsService -from aleph.services.storage.engine import StorageEngine - from aleph.exceptions import InvalidContent from aleph.schemas.message_content import ContentSource from aleph.schemas.pending_messages import parse_message +from aleph.services.ipfs import IpfsService +from aleph.services.storage.engine import StorageEngine from aleph.storage import StorageService diff --git a/tests/storage/test_store_message.py b/tests/storage/test_store_message.py index c0ac36797..00e029322 100644 --- a/tests/storage/test_store_message.py +++ b/tests/storage/test_store_message.py @@ -1,8 +1,9 @@ import json -from typing import Mapping, Any +from typing import Any, Mapping import pytest from configmanager import Config +from message_test_helpers import make_validated_message_from_dict from sqlalchemy import select from aleph.db.models import MessageDb, StoredFileDb @@ -12,7 +13,6 @@ from aleph.storage import StorageService from aleph.types.db_session import DbSessionFactory from aleph.types.files import FileType -from message_test_helpers import make_validated_message_from_dict @pytest.fixture diff --git a/tests/test_network.py b/tests/test_network.py index 886eeff87..9ae83e6b5 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -3,7 +3,6 @@ import pytest from configmanager import Config -from aleph.chains.connector import ChainConnector from aleph.chains.signature_verifier import SignatureVerifier from aleph.db.models import PendingMessageDb from aleph.handlers.message_handler import MessageHandler diff --git a/tests/toolkit/test_batch.py b/tests/toolkit/test_batch.py index 8e7c0fa73..0f04adf44 100644 --- a/tests/toolkit/test_batch.py +++ b/tests/toolkit/test_batch.py @@ -1,4 +1,5 @@ import pytest + from aleph.toolkit.batch import async_batch diff --git a/tests/toolkit/test_range.py b/tests/toolkit/test_range.py index a3410ad60..f270a81d9 100644 --- a/tests/toolkit/test_range.py +++ b/tests/toolkit/test_range.py @@ -1,8 +1,9 @@ from typing import List -from aleph.toolkit.range import Range, MultiRange, int_range import pytest +from aleph.toolkit.range import MultiRange, Range, int_range + @pytest.mark.parametrize( "range_str, expected", diff --git a/tests/toolkit/test_timer.py b/tests/toolkit/test_timer.py index 7f0e5a5a5..e1fd861b9 100644 --- a/tests/toolkit/test_timer.py +++ b/tests/toolkit/test_timer.py @@ -1,6 +1,7 @@ -from aleph.toolkit.timer import Timer import time +from aleph.toolkit.timer import Timer + def test_timer_sleep(): sleep_duration = 0.5 diff --git a/tests/web/controllers/test_metrics.py b/tests/web/controllers/test_metrics.py index 6ec4ea3a2..cc7989baa 100644 --- a/tests/web/controllers/test_metrics.py +++ b/tests/web/controllers/test_metrics.py @@ -1,17 +1,24 @@ from dataclasses import dataclass -from aleph.web.controllers.metrics import format_dict_for_prometheus, \ - format_dataclass_for_prometheus, Metrics, BuildInfo +from aleph.web.controllers.metrics import ( + BuildInfo, + Metrics, + format_dataclass_for_prometheus, + format_dict_for_prometheus, +) def test_format_dict_for_prometheus(): - assert format_dict_for_prometheus( - { - 'a': 1, - 'b': 2.2, - 'c': "3", - } - ) == '{a=1,b=2.2,c="3"}' + assert ( + format_dict_for_prometheus( + { + "a": 1, + "b": 2.2, + "c": "3", + } + ) + == '{a=1,b=2.2,c="3"}' + ) def test_format_dataclass_for_prometheus() -> None: @@ -22,26 +29,24 @@ class Simple: b: float c: str - assert format_dataclass_for_prometheus( - Simple(1, 2.2, "3") - ) == 'a 1\nb 2.2\nc "3"' + assert format_dataclass_for_prometheus(Simple(1, 2.2, "3")) == 'a 1\nb 2.2\nc "3"' @dataclass class Tagged: d: Simple e: str - - assert format_dataclass_for_prometheus( - Tagged(Simple(1, 2.2, "3"), "e") - ) == 'd{a=1,b=2.2,c="3"} 1\ne "e"' + assert ( + format_dataclass_for_prometheus(Tagged(Simple(1, 2.2, "3"), "e")) + == 'd{a=1,b=2.2,c="3"} 1\ne "e"' + ) def test_metrics(): metrics = Metrics( pyaleph_build_info=BuildInfo( - python_version='3.8.0', - version='v999', + python_version="3.8.0", + version="v999", ), pyaleph_status_peers_total=0, pyaleph_status_sync_messages_total=123, @@ -56,18 +61,17 @@ def test_metrics(): pyaleph_processing_pending_messages_store_tasks=0, ) - assert format_dataclass_for_prometheus( - metrics - ) == ('pyaleph_build_info{python_version="3.8.0",version="v999"} 1\n' - 'pyaleph_status_peers_total 0\n' - 'pyaleph_status_sync_messages_total 123\n' - 'pyaleph_status_sync_permanent_files_total 1999\n' - 'pyaleph_status_sync_pending_messages_total 456\n' - 'pyaleph_status_sync_pending_txs_total 0\n' - 'pyaleph_status_chain_eth_last_committed_height 0\n' - 'pyaleph_processing_pending_messages_aggregate_tasks 0\n' - 'pyaleph_processing_pending_messages_forget_tasks 0\n' - 'pyaleph_processing_pending_messages_post_tasks 0\n' - 'pyaleph_processing_pending_messages_program_tasks 0\n' - 'pyaleph_processing_pending_messages_store_tasks 0' + assert format_dataclass_for_prometheus(metrics) == ( + 'pyaleph_build_info{python_version="3.8.0",version="v999"} 1\n' + "pyaleph_status_peers_total 0\n" + "pyaleph_status_sync_messages_total 123\n" + "pyaleph_status_sync_permanent_files_total 1999\n" + "pyaleph_status_sync_pending_messages_total 456\n" + "pyaleph_status_sync_pending_txs_total 0\n" + "pyaleph_status_chain_eth_last_committed_height 0\n" + "pyaleph_processing_pending_messages_aggregate_tasks 0\n" + "pyaleph_processing_pending_messages_forget_tasks 0\n" + "pyaleph_processing_pending_messages_post_tasks 0\n" + "pyaleph_processing_pending_messages_program_tasks 0\n" + "pyaleph_processing_pending_messages_store_tasks 0" ) diff --git a/tests/web/controllers/test_programs.py b/tests/web/controllers/test_programs.py index 62d5b0690..b796c81f1 100644 --- a/tests/web/controllers/test_programs.py +++ b/tests/web/controllers/test_programs.py @@ -5,10 +5,10 @@ import pytest import pytest_asyncio +from message_test_helpers import make_validated_message_from_dict from aleph.db.models import MessageDb from aleph.types.db_session import DbSessionFactory -from message_test_helpers import make_validated_message_from_dict @pytest_asyncio.fixture