Merge branch 'main' into hoh-time-datetime

.github/workflows/tests.yml

@@ -1,13 +1,19 @@
 name: Tests
 
-on: push
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
 
 jobs:
   build:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Use GitHub's Docker registry to cache intermediate layers
       - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u $GITHUB_ACTOR --password-stdin
@@ -23,4 +29,8 @@ jobs:
 
       - name: Pytest in the Docker image
         run: |
-          docker run aleph-message:${GITHUB_REF##*/} pytest -vv
+          docker run aleph-message:${GITHUB_REF##*/} pytest -vv --cov aleph_message
+
+      - name: MyPy in the Docker image
+        run: |
+          docker run aleph-message:${GITHUB_REF##*/} mypy aleph_message --ignore-missing-imports --check-untyped-defs

README.md

@@ -1,31 +1,43 @@
 # Aleph.im Message Specification
 
-This library aims to provide an easy way to create, update and validate 
+This library aims to provide an easy way to create, update and manipulate 
 messages from Aleph.im.
 
 It mainly consists in [pydantic](https://pydantic-docs.helpmanual.io/) 
 models that provide field type validation and IDE autocompletion for messages.
 
-## Status
+This library provides:
+* schema validation when parsing messages.
+* cryptographic hash validation that the `item_hash` matches the content of the message.
+* type validation using type checkers such as [mypy](https://www.mypy-lang.org/) in development environments.
+* autocompletion support in development editors.
 
-Currently, only basic type validation is included. Advanced data and signature
-validation is not included.
+The `item_hash` is commonly used as unique message identifier on Aleph.im.
 
-In the future, this library would be useful within other projects such as
-the client library [aleph-client](https://github.com/aleph-im/aleph-client).
+Cryptographic signatures are out of scope of this library and part of the `aleph-sdk-python`
+project, due to their extended scope and dependency on cryptographic libraries.
+
+This library is used in both client and node software of Aleph.im.
 
 ## Usage
 
+```shell
+pip install aleph-message
+```
+
 ```python
 import requests
-from aleph_message import Message
+from aleph_message import parse_message
 from pydantic import ValidationError
 
-message_dict = requests.get(ALEPH_API_SERVER + "/api/v0/messages.json?hashes=...").json()
+ALEPH_API_SERVER = "https://official.aleph.cloud"
+MESSAGE_ITEM_HASH = "9b21eb870d01bf64d23e1d4475e342c8f958fcd544adc37db07d8281da070b00"
+
+message_dict = requests.get(ALEPH_API_SERVER + "/api/v0/messages.json?hashes=" + MESSAGE_ITEM_HASH).json()
 
 try:
-    message = Message(**message_dict)
+    message = parse_message(message_dict["messages"][0])
     print(message.sender)
 except ValidationError as e:
-    print(e.json())
+    print(e.json(indent=4))
 ```

aleph_message/__init__.py

@@ -1 +1,4 @@
-from .models import Message, MessagesResponse
+from .models import MessagesResponse, parse_message
+
+__all__ = ["parse_message", "MessagesResponse"]
+__version__ = "0.4.0"

aleph_message/models/__init__.py

@@ -5,25 +5,30 @@
 from hashlib import sha256
 from json import JSONDecodeError
 from pathlib import Path
-from typing import List, Dict, Any, Optional, Union, NewType
-
-from .item_hash import ItemHash, ItemType
-
-try:
-    from typing import Literal
-except ImportError:
-    from typing_extensions import Literal
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
 
 from pydantic import BaseModel, Extra, Field, validator
+from typing_extensions import TypeAlias
 
 from .abstract import BaseContent
-from .program import ProgramContent
+from .execution.instance import InstanceContent
+from .execution.program import ProgramContent
+from .item_hash import ItemHash, ItemType
 
 
 class Chain(str, Enum):
     """Supported chains"""
 
     AVAX = "AVAX"
+    BSC = "BSC"
     CSDK = "CSDK"
     DOT = "DOT"
     ETH = "ETH"
@@ -47,6 +52,7 @@ class MessageType(str, Enum):
     aggregate = "AGGREGATE"
     store = "STORE"
     program = "PROGRAM"
+    instance = "INSTANCE"
     forget = "FORGET"
 
 
@@ -90,7 +96,9 @@ class MessageConfirmation(BaseModel):
     # These two optional fields are introduced in recent versions of CCNs. They should
     # remain optional until the corresponding CCN upload (0.4.0) is widely uploaded.
     time: Optional[float] = None
-    publisher: Optional[str] = Field(default=None, description="The address that published the transaction.")
+    publisher: Optional[str] = Field(
+        default=None, description="The address that published the transaction."
+    )
 
     class Config:
         extra = Extra.forbid
@@ -170,7 +178,10 @@ class BaseMessage(BaseModel):
     """Base template for all messages"""
 
     id_: Optional[MongodbId] = Field(
-        alias="_id", default=None, description="MongoDB metadata"
+        alias="_id",
+        default=None,
+        description="MongoDB metadata",
+        exclude=True,
     )
     chain: Chain = Field(description="Blockchain used for this message")
 
@@ -187,7 +198,7 @@ class BaseMessage(BaseModel):
         default=None,
         description="Indicates that the message has been confirmed on a blockchain",
     )
-    signature: str = Field(
+    signature: Optional[str] = Field(
         description="Cryptographic signature of the message by the sender"
     )
     size: Optional[int] = Field(
@@ -210,18 +221,19 @@ class BaseMessage(BaseModel):
     @validator("item_content")
     def check_item_content(cls, v: Optional[str], values):
         item_type = values["item_type"]
-        if item_type == ItemType.inline:
+        if v is None:
+            return None
+        elif item_type == ItemType.inline:
             try:
                 json.loads(v)
             except JSONDecodeError:
                 raise ValueError(
                     "Field 'item_content' does not appear to be valid JSON"
                 )
         else:
-            if v is not None:
-                raise ValueError(
-                    f"Field 'item_content' cannot be defined when 'item_type' == '{item_type}'"
-                )
+            raise ValueError(
+                f"Field 'item_content' cannot be defined when 'item_type' == '{item_type}'"
+            )
         return v
 
     @validator("item_hash")
@@ -250,7 +262,7 @@ def check_item_hash(cls, v, values):
     @validator("confirmed")
     def check_confirmed(cls, v, values):
         confirmations = values["confirmations"]
-        if v != bool(confirmations):
+        if v is True and not bool(confirmations):
             raise ValueError("Message cannot be 'confirmed' without 'confirmations'")
         return v
 
@@ -263,6 +275,7 @@ def check_time(cls, v, values):
 
     class Config:
         extra = Extra.forbid
+        exclude = {"id_", "_id"}
 
 
 class PostMessage(BaseMessage):
@@ -315,25 +328,50 @@ def check_content(cls, v, values):
         return v
 
 
-message_types = (
+class InstanceMessage(BaseMessage):
+    type: Literal[MessageType.instance]
+    content: InstanceContent
+
+
+AlephMessage: TypeAlias = Union[
     PostMessage,
     AggregateMessage,
     StoreMessage,
     ProgramMessage,
+    InstanceMessage,
     ForgetMessage,
-)
+]
+
+AlephMessageType: TypeAlias = Union[
+    Type[PostMessage],
+    Type[AggregateMessage],
+    Type[StoreMessage],
+    Type[ProgramMessage],
+    Type[InstanceMessage],
+    Type[ForgetMessage],
+]
+
+message_classes: List[AlephMessageType] = [
+    PostMessage,
+    AggregateMessage,
+    StoreMessage,
+    ProgramMessage,
+    InstanceMessage,
+    ForgetMessage,
+]
 
-AlephMessage = NewType("AlephMessage", Union[message_types])
+ExecutableContent: TypeAlias = Union[InstanceContent, ProgramContent]
+ExecutableMessage: TypeAlias = Union[InstanceMessage, ProgramMessage]
 
 
-def Message(**message_dict: Dict) -> AlephMessage:
+def parse_message(message_dict: Dict) -> AlephMessage:
     """Returns the message class corresponding to the type of message."""
-    for message_class in message_types:
+    for message_class in message_classes:
         message_type: MessageType = MessageType(
             message_class.__annotations__["type"].__args__[0]
         )
         if message_dict["type"] == message_type:
-            return message_class(**message_dict)
+            return message_class.parse_obj(message_dict)
     else:
         raise ValueError(f"Unknown message type {message_dict['type']}")
 
@@ -352,27 +390,36 @@ def add_item_content_and_hash(message_dict: Dict, inplace: bool = False):
 
 
 def create_new_message(
-    message_dict: Dict, factory: Union[Message, AlephMessage] = Message
+    message_dict: Dict,
+    factory: Optional[AlephMessageType] = None,
 ) -> AlephMessage:
     """Create a new message from a dict.
     Computes the 'item_content' and 'item_hash' fields.
     """
-    return factory(**add_item_content_and_hash(message_dict))
+    message_content = add_item_content_and_hash(message_dict)
+    if factory:
+        return factory.parse_obj(message_content)
+    else:
+        return parse_message(message_content)
 
 
 def create_message_from_json(
-    json_data: str, factory: Union[Message, AlephMessage] = Message
+    json_data: str,
+    factory: Optional[AlephMessageType] = None,
 ) -> AlephMessage:
     """Create a new message from a JSON encoded string.
     Computes the 'item_content' and 'item_hash' fields.
     """
     message_dict = json.loads(json_data)
-    add_item_content_and_hash(message_dict, inplace=True)
-    return factory(**message_dict)
+    message_content = add_item_content_and_hash(message_dict, inplace=True)
+    if factory:
+        return factory.parse_obj(message_content)
+    else:
+        return parse_message(message_content)
 
 
 def create_message_from_file(
-    filepath: Path, factory: Union[Message, AlephMessage] = Message, decoder=json
+    filepath: Path, factory: Optional[AlephMessageType] = None, decoder=json
 ) -> AlephMessage:
     """Create a new message from an encoded file.
     Expects json by default, but allows other decoders with a method `.load()`
@@ -381,8 +428,11 @@ def create_message_from_file(
     """
     with open(filepath) as fd:
         message_dict = decoder.load(fd)
-    add_item_content_and_hash(message_dict, inplace=True)
-    return factory(**message_dict)
+    message_content = add_item_content_and_hash(message_dict, inplace=True)
+    if factory:
+        return factory.parse_obj(message_content)
+    else:
+        return parse_message(message_content)
 
 
 class MessagesResponse(BaseModel):

aleph_message/models/execution/__init__.py

@@ -0,0 +1,5 @@
+from .abstract import BaseExecutableContent
+from .instance import InstanceContent
+from .program import ProgramContent
+
+__all__ = ["BaseExecutableContent", "InstanceContent", "ProgramContent"]

aleph_message/models/execution/abstract.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from abc import ABC
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field
+
+from .environment import (
+    FunctionEnvironment,
+    HostRequirements,
+    MachineResources,
+)
+from .volume import MachineVolume
+from ..abstract import BaseContent, HashableModel
+
+
+class BaseExecutableContent(HashableModel, BaseContent, ABC):
+    """Abstract content for execution messages (Instances, Programs)."""
+
+    allow_amend: bool = Field(description="Allow amends to update this function")
+    metadata: Optional[Dict[str, Any]] = Field(description="Metadata of the VM")
+    authorized_keys: Optional[List[str]] = Field(
+        description="SSH public keys authorized to connect to the VM",
+    )
+    variables: Optional[Dict[str, str]] = Field(
+        default=None, description="Environment variables available in the VM"
+    )
+    environment: FunctionEnvironment = Field(
+        description="Properties of the execution environment"
+    )
+    resources: MachineResources = Field(description="System resources required")
+    requirements: Optional[HostRequirements] = Field(
+        default=None, description="System properties required"
+    )
+    volumes: List[MachineVolume] = Field(
+        default=[], description="Volumes to mount on the filesystem"
+    )
+    replaces: Optional[str] = Field(
+        default=None,
+        description="Previous version to replace. Must be signed by the same address",
+    )

aleph_message/models/execution/base.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class Encoding(str, Enum):
+    """Code and data can be provided in plain format, as zip or as squashfs partition."""
+
+    plain = "plain"
+    zip = "zip"
+    squashfs = "squashfs"
+
+
+class MachineType(str, Enum):
+    """Two types of execution environments supported:
+    Instance (Virtual Private Server) and Function (Program oriented)."""
+
+    vm_instance = "vm-instance"
+    vm_function = "vm-function"