diff --git a/CHANGELOG.md b/CHANGELOG.md index 50f23e5..c351903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Mediagrains Library Changelog +## 2.7.0 +- Dropped all support for Python2.7 +- Moved python3.6 specific submodules in tree +- Added `GrainWrapper` class to wrap raw essence in Grains. +- Added `wrap_video_in_gsf` and `wrap_audio_in_gsf` tools to generate GSF files from raw essence. +- Added `extract_from_gsf` and `gsf_probe` tools to extract essence and metadata from GSF files. +- Added MyPy as a dependency +- Deprecated old asyncio code from v2.6 +- Added Asynchronous GSFEncoding using the standard Encoder in a context-manager type workflow. +- Added Asynchronous GSFDecoding using the standard Decoder in a context-manager type workflow. + ## 2.6.0 - Added support for async methods to gsf decoder in python 3.6+ - Added `Grain.origin_timerange` method. diff --git a/Jenkinsfile b/Jenkinsfile index b6f406c..9f16c1a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,9 @@ @Library("rd-apmm-groovy-ci-library@v1.x") _ /* - Runs the following steps in parallel and reports results to GitHub: + Runs the following steps in series and reports results to GitHub: - Lint using flake8 - - Run Python 2.7 unit tests in tox + - Type check using mypy - Run Python 3 unit tests in tox - Build Debian packages for supported Ubuntu versions @@ -44,84 +44,62 @@ pipeline { } stage ("Tests") { stages { - stage ("Py2.7 Linting Check") { + stage ("Py36 Linting Check") { steps { script { - env.lint27_result = "FAILURE" + env.lint3_result = "FAILURE" } - bbcGithubNotify(context: "lint/flake8_27", status: "PENDING") + bbcGithubNotify(context: "lint/flake8_3", status: "PENDING") // Run the linter - sh 'python2.7 -m flake8 --filename=mediagrains/*.py,tests/test_*.py' + sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-lint make lint' script { - env.lint27_result = "SUCCESS" // This will only run if the sh above succeeded + env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded } } post { always { - bbcGithubNotify(context: "lint/flake8_27", status: env.lint27_result) + bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result) } } } - stage ("Py36 Linting Check") { + stage ("Py36 Type Check") { steps { script { - env.lint3_result = "FAILURE" + env.mypy_result = "FAILURE" } - bbcGithubNotify(context: "lint/flake8_3", status: "PENDING") + bbcGithubNotify(context: "type/mypy", status: "PENDING") // Run the linter - sh 'python3 -m flake8 --filename=mediagrains/*.py,mediagrains_async/*.py,tests/test_*.py,tests/atest_*.py' + sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-mypy make mypy' script { - env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded + env.mypy_result = "SUCCESS" // This will only run if the sh above succeeded } } post { always { - bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result) + bbcGithubNotify(context: "type/mypy", status: env.mypy_result) } } } stage ("Build Docs") { - steps { - sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs' - } + steps { + sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs' + } } - stage ("Unit Tests") { - stages { - stage ("Python 2.7 Unit Tests") { - steps { - script { - env.py27_result = "FAILURE" - } - bbcGithubNotify(context: "tests/py27", status: "PENDING") - // Use a workdirectory in /tmp to avoid shebang length limitation - sh 'tox -e py27 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py27' - script { - env.py27_result = "SUCCESS" // This will only run if the sh above succeeded - } - } - post { - always { - bbcGithubNotify(context: "tests/py27", status: env.py27_result) - } - } + stage ("Python 3 Unit Tests") { + steps { + script { + env.py36_result = "FAILURE" } - stage ("Python 3 Unit Tests") { - steps { - script { - env.py36_result = "FAILURE" - } - bbcGithubNotify(context: "tests/py36", status: "PENDING") - // Use a workdirectory in /tmp to avoid shebang length limitation - sh 'tox -e py36 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py36' - script { - env.py36_result = "SUCCESS" // This will only run if the sh above succeeded - } - } - post { - always { - bbcGithubNotify(context: "tests/py36", status: env.py36_result) - } - } + bbcGithubNotify(context: "tests/py36", status: "PENDING") + // Use a workdirectory in /tmp to avoid shebang length limitation + sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-py36 make test' + script { + env.py36_result = "SUCCESS" // This will only run if the sh above succeeded + } + } + post { + always { + bbcGithubNotify(context: "tests/py36", status: env.py36_result) } } } @@ -218,7 +196,6 @@ pipeline { } bbcGithubNotify(context: "pypi/upload", status: "PENDING") sh 'rm -rf dist/*' - bbcMakeGlobalWheel("py27") bbcMakeGlobalWheel("py36") bbcTwineUpload(toxenv: "py36", pypi: true) script { @@ -246,7 +223,6 @@ pipeline { } bbcGithubNotify(context: "artifactory/upload", status: "PENDING") sh 'rm -rf dist/*' - bbcMakeGlobalWheel("py27") bbcMakeGlobalWheel("py36") bbcTwineUpload(toxenv: "py36", pypi: false) script { @@ -276,9 +252,9 @@ pipeline { script { for (def dist in bbcGetSupportedUbuntuVersions()) { bbcDebUpload(sourceFiles: "_result/${dist}-amd64/*", - removePrefix: "_result/${dist}-amd64", - dist: "${dist}", - apt_repo: "ap/python") + removePrefix: "_result/${dist}-amd64", + dist: "${dist}", + apt_repo: "ap/python") } } script { diff --git a/MANIFEST.in b/MANIFEST.in index f455058..55cbb0c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,5 @@ include tox.ini include COPYING recursive-include examples *.gsf recursive-include tests *.py -recursive-include mediagrains_py36 *.py include ICLA.md include LICENSE.md diff --git a/Makefile b/Makefile index 4a81427..56fbf47 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,7 @@ -PYTHON=`which python` -PYTHON2=`which python2` PYTHON3=`which python3` PY2DSC=`which py2dsc` -PY2DSC_PARAMS?=--with-python2=true --with-python3=true +PY2DSC_PARAMS?=--with-python2=false --with-python3=true topdir := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) topbuilddir := $(realpath .) @@ -15,7 +13,7 @@ MODNAME=$(PROJECT) # The rules for names and versions in python, rpm, and deb are different # and not entirely compatible. As such py2dsc will automatically convert -# your package name into a suitable deb name and version number, and this +# your package name into a suitable deb name and version number, and this # code replicates that. DEBNAME=$(shell echo $(MODNAME) | tr '[:upper:]_' '[:lower:]-') DEBVERSION=$(shell echo $(VERSION) | sed 's/\.dev/~dev/') @@ -49,15 +47,15 @@ $(topbuilddir)/dist: mkdir -p $@ source: $(topbuilddir)/dist - $(PYTHON) $(topdir)/setup.py sdist $(COMPILE) --dist-dir=$(topbuilddir)/dist + $(PYTHON3) $(topdir)/setup.py sdist $(COMPILE) --dist-dir=$(topbuilddir)/dist $(topbuilddir)/dist/$(MODNAME)-$(VERSION).tar.gz: source install: - $(PYTHON) $(topdir)/setup.py install --root $(DESTDIR) $(COMPILE) + $(PYTHON3) $(topdir)/setup.py install --root $(DESTDIR) $(COMPILE) clean: - $(PYTHON) $(topdir)/setup.py clean || true + $(PYTHON3) $(topdir)/setup.py clean || true rm -rf $(topbuilddir)/.tox rm -rf $(topbuilddir)/build/ MANIFEST rm -rf $(topbuilddir)/dist @@ -67,13 +65,10 @@ clean: find $(topdir) -name '*.py,cover' -delete rm -rf $(topbuilddir)/docs -testenv: $(TOXDIR)/py27/bin/activate $(TOXDIR)/py3/bin/activate +testenv: $(TOXDIR)/py36/bin/activate -$(TOXDIR)/py3/bin/activate: tox.ini - tox -e py3 --recreate --workdir $(TOXDIR) - -$(TOXDIR)/py27/bin/activate: tox.ini - tox -e py27 --recreate --workdir $(TOXDIR) +$(TOXDIR)/py36/bin/activate: tox.ini + tox -e py36 --recreate --notest --workdir $(TOXDIR) test: tox --workdir $(TOXDIR) @@ -117,11 +112,9 @@ rpm: $(RPM_PREFIX)/SPECS/$(MODNAME).spec $(RPM_PREFIX)/SOURCES/$(MODNAME)-$(VERS cp $(RPM_PREFIX)/RPMS/*/*.rpm $(topbuilddir)/dist wheel: - $(PYTHON2) $(topdir)/setup.py bdist_wheel $(PYTHON3) $(topdir)/setup.py bdist_wheel egg: - $(PYTHON2) $(topdir)/setup.py bdist_egg $(PYTHON3) $(topdir)/setup.py bdist_egg docs: $(topbuilddir)/docs/$(MODNAME).html @@ -130,7 +123,13 @@ $(topbuilddir)/docs/$(MODNAME): mkdir -p $(topbuilddir)/docs ln -s $(topdir)/$(MODNAME) $(topbuilddir)/docs/ -$(topbuilddir)/docs/$(MODNAME).html: $(topbuilddir)/docs/$(MODNAME) $(TOXDIR)/py3/bin/activate - . $(TOXDIR)/py3/bin/activate && cd $(topbuilddir)/docs/ && pydoc -w ./ +$(topbuilddir)/docs/$(MODNAME).html: $(topbuilddir)/docs/$(MODNAME) $(TOXDIR)/py36/bin/activate + . $(TOXDIR)/py36/bin/activate && cd $(topbuilddir)/docs/ && pydoc -w ./ + +lint: $(TOXDIR)/py36/bin/activate + . $(TOXDIR)/py36/bin/activate && python -m flake8 + +mypy: $(TOXDIR)/py36/bin/activate + . $(TOXDIR)/py36/bin/activate && python -m mypy -p $(MODNAME) -.PHONY: test testenv clean install source deb dsc rpm wheel egg all rpm_dirs rpm_spec docs +.PHONY: test testenv clean install source deb dsc rpm wheel egg all rpm_dirs rpm_spec docs lint mypy diff --git a/README.md b/README.md index 938a372..1c84bf9 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,14 @@ that nicely wrap those grains, as well as a full serialisation and deserialisation library for GSF format. Please read the pydoc documentation for more details. +Some useful tools for handling the Grain Sequence Format (GSF) file format +are also included - see [Tools](#tools). + ## Installation ### Requirements -* A working Python 2.7 or Python 3.6+ installation +* A working Python 3.6+ installation * BBC R&D's internal deb repository set up as a source for apt (if installing via apt-get) * The tool [tox](https://tox.readthedocs.io/en/latest/) is needed to run the unittests, but not required to use the library. @@ -25,7 +28,7 @@ documentation for more details. $ pip install mediagrains # Install via apt-get -$ apt-get install python-mediagrains python3-mediagrains +$ apt-get install python3-mediagrains # Install directly from source repo $ git clone git@github.com:bbc/rd-apmm-python-lib-mediagrains.git @@ -85,7 +88,7 @@ it with colour-bars: ... i += 1 ``` -(In python3.6+ a more natural interface for accessing data exists in the form of numpy arrays. See later.) +(a more natural interface for accessing data exists in the form of numpy arrays. See later.) The object grain can then be freely used for whatever video processing is desired, or it can be serialised into a GSF file as follows: @@ -161,9 +164,9 @@ between two grains, both as a printed string (as seen above) and also in a data-centric fashion as a tree structure which can be interrogated in code. -### Numpy arrays (Python 3.6+) +### Numpy arrays -In python 3.6 or higher an additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily: +An additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily: ```Python console >>> from mediagrains.numpy import VideoGrain @@ -193,6 +196,30 @@ The API is well documented in the docstrings of the module mediagrains, to view: pydoc mediagrains ``` +## Tools +Some tools are installed with the library to make working with the Grain Sequence Format (GSF) file format easier. + +* `wrap_video_in_gsf` - Provides a means to read raw video essence and generate a GSF file. +* `wrap_audio_in_gsf` - As above, but for audio. +* `extract_from_gsf` - Read a GSF file and dump out the raw essence within. +* `gsf_probe` - Read metadata about the segments in a GSF file. + +For example, to generate a GSF file containing a test pattern from `ffmpeg`, dump the metadata and then play it out +again: +```bash +ffmpeg -f lavfi -i testsrc=duration=20:size=1920x1080:rate=25 -pix_fmt yuv422p10le -c:v rawvideo -f rawvideo - | \ +wrap_video_in_gsf - output.gsf --size 1920x1080 --format S16_422_10BIT --rate 25 +gsf_probe output.gsf +extract_gsf_essence output.gsf - | ffplay -f rawvideo -pixel_format yuv422p10 -video_size 1920x1080 -framerate 25 pipe:0 +``` + +To do the same with a sine wave: +```bash +ffmpeg -f lavfi -i "sine=frequency=1000:duration=5" -f s16le -ac 2 - | wrap_audio_in_gsf - output_audio.gsf --sample-rate 44100 +gsf_probe output_audio.gsf +extract_gsf_essence output_audio.gsf - | ffplay -f s16le -ac 2 -ar 44100 pipe:0 +``` + ## Development ### Testing diff --git a/mediagrains/__init__.py b/mediagrains/__init__.py index f9659c4..81f14dc 100644 --- a/mediagrains/__init__.py +++ b/mediagrains/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -40,7 +39,6 @@ The gsf and grain submodules have their own documentation. """ -from __future__ import absolute_import from .grain_constructors import Grain, VideoGrain, CodedVideoGrain, AudioGrain, CodedAudioGrain, EventGrain __all__ = ["Grain", "VideoGrain", "CodedVideoGrain", "AudioGrain", "CodedAudioGrain", "EventGrain"] diff --git a/mediagrains/asyncio.py b/mediagrains/asyncio.py deleted file mode 100644 index 2ad0429..0000000 --- a/mediagrains/asyncio.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2019 British Broadcasting Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""\ -Asyncio compatible layer for mediagrains, but only available in python 3.6+ -""" - -from sys import version_info - -if version_info[0] > 3 or (version_info[0] == 3 and version_info[1] >= 6): - from mediagrains_py36.asyncio import AsyncGSFDecoder, AsyncLazyLoaderUnloadedError, loads # noqa: F401 - - __all__ = ["AsyncGSFDecoder", "AsyncLazyLoaderUnloadedError", "loads"] -else: - __all__ = [] diff --git a/mediagrains_py36/asyncio/__init__.py b/mediagrains/asyncio/__init__.py similarity index 90% rename from mediagrains_py36/asyncio/__init__.py rename to mediagrains/asyncio/__init__.py index 9d8be9f..fe3cb04 100644 --- a/mediagrains_py36/asyncio/__init__.py +++ b/mediagrains/asyncio/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2019 British Broadcasting Corporation # @@ -17,8 +16,16 @@ """\ Library for handling mediagrains in pure python asyncio compatibility layer. + +THIS SUBLIBRARY IS DEPRECATED. + +DO NOT USE IT IN NEW WORK, IT WILL SOON BE REMOVED. + +THE ASYNCIO CAPABILITIES ARE NOW INCLUDED IN mediagrains.gsf """ +from deprecated import deprecated + import asyncio from uuid import UUID @@ -37,6 +44,7 @@ __all__ = ["AsyncGSFDecoder", "AsyncLazyLoaderUnloadedError", "loads"] +@deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def loads(s, cls=None, parse_grain=None, **kwargs): """Deserialise a GSF file from a string (or similar) into python, returns a pair of (head, segments) where head is a python dict @@ -68,6 +76,7 @@ class AsyncGSFBlock(): Must be used as an asynchronous context manager, which will automatically decode the block tag and size, exposed by the `tag` and `size` attributes. """ + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") def __init__(self, file_data, want_tag=None, raise_on_wrong_tag=False): """Constructor. Unlike the synchronous version does not record the start byte of the block in `block_start` @@ -82,6 +91,7 @@ def __init__(self, file_data, want_tag=None, raise_on_wrong_tag=False): self.size = None self.block_start = None + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def __aenter__(self): """When used as a context manager record file position and read block size and tag on entry @@ -117,10 +127,12 @@ async def __aenter__(self): await self.file_data.seek(self.block_start + self.size, SEEK_SET) self.block_start = await self.file_data.tell() + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def __aexit__(self, *args): """When used as a context manager, exiting context should seek to the block end""" await self.file_data.seek(self.block_start + self.size, SEEK_SET) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def has_child_block(self, strict_blocks=True): """Checks if there is space for another child block in this block @@ -145,6 +157,7 @@ async def has_child_block(self, strict_blocks=True): else: return False + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def child_blocks(self, strict_blocks=True): """Asynchronous generator for each child block - each yielded block sits within the context manager @@ -158,6 +171,7 @@ async def child_blocks(self, strict_blocks=True): async with AsyncGSFBlock(self.file_data) as child_block: yield child_block + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def get_remaining(self): """Get the number of bytes left in this block @@ -168,6 +182,7 @@ async def get_remaining(self): assert self.size is not None, "get_remaining() only works in a context manager" return (self.block_start + self.size) - await self.file_data.tell() + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_uint(self, length): """Read an unsigned integer of length `length` @@ -185,6 +200,7 @@ async def read_uint(self, length): r += (uint_bytes[n] << (n*8)) return r + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_bool(self): """Read a boolean value @@ -193,6 +209,7 @@ async def read_bool(self): n = await self.read_uint(1) return (n != 0) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_sint(self, length): """Read a 2's complement signed integer @@ -205,6 +222,7 @@ async def read_sint(self, length): r -= (1 << (8*length)) return r + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_string(self, length): """Read a fixed-length string, treating it as UTF-8 @@ -218,6 +236,7 @@ async def read_string(self, length): return string_data.decode(encoding='utf-8') + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_varstring(self): """Read a variable length string @@ -229,6 +248,7 @@ async def read_varstring(self): length = await self.read_uint(2) return await self.read_string(length) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_uuid(self): """Read a UUID @@ -242,6 +262,7 @@ async def read_uuid(self): return UUID(bytes=uuid_data) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_timestamp(self): """Read a date-time (with seconds resolution) stored in 7 bytes @@ -256,6 +277,7 @@ async def read_timestamp(self): second = await self.read_uint(1) return datetime(year, month, day, hour, minute, second) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_ippts(self): """Read a mediatimestamp.Timestamp @@ -266,6 +288,7 @@ async def read_ippts(self): nano = await self.read_uint(4) return Timestamp(secs, nano) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def read_rational(self): """Read a rational (fraction) @@ -282,6 +305,7 @@ async def read_rational(self): return Fraction(numerator, denominator) +@deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") def asynchronise(f): async def __inner(*args, **kwargs): return f(*args, **kwargs) @@ -294,6 +318,7 @@ class AsyncGSFDecoder(object): Provides coroutines to decode the header of a GSF file, followed by an asynchronous generator to get each grain, wrapped in some grain method (mediagrains.Grain by default.) """ + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") def __init__(self, file_data, parse_grain=Grain, **kwargs): """Constructor @@ -308,6 +333,7 @@ def __init__(self, file_data, parse_grain=Grain, **kwargs): self.head = None self.start_loc = None + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def _decode_ssb_header(self): """Find and read the SSB header in the GSF file @@ -327,6 +353,7 @@ async def _decode_ssb_header(self): return (major, minor) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def _decode_head(self, head_block): """Decode the "head" block and extract ID, created date, segments and tags @@ -368,6 +395,7 @@ async def _decode_head(self, head_block): return head + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def _decode_tils(self, tils_block): """Decode timelabels (tils) block @@ -391,6 +419,7 @@ async def _decode_tils(self, tils_block): return tils + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def _decode_gbhd(self, gbhd_block): """Decode grain block header ("gbhd") to get grain metadata @@ -504,6 +533,7 @@ async def _decode_gbhd(self, gbhd_block): return meta + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def decode_file_headers(self): """Verify the file is a supported version, and get the file header @@ -526,20 +556,24 @@ async def decode_file_headers(self): except EOFError: raise GSFDecodeError("No head block found in file", await self.file_data.tell()) + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def __aenter__(self): if self.start_loc is None: self.start_loc = await self.file_data.tell() await self.decode_file_headers() return self + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def __aexit__(self, *args, **kwargs): if self.start_loc is not None: await self.file_data.seek(self.start_loc) self.start_loc = None + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") def __aiter__(self): return self.grains() + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def grains(self, local_ids=None, load_lazily=True): """Asynchronous generator to get grains from the GSF file. Skips blocks which aren't "grai". @@ -587,6 +621,7 @@ async def grains(self, local_ids=None, load_lazily=True): except EOFError: return # We ran out of grains to read and hit EOF + @deprecated(version="2.7.0", reason="Asyncio is now supported directly in mediagrains.gsf") async def decode(self, load_lazily=False): """Decode a GSF formatted bytes object diff --git a/mediagrains_py36/asyncio/aiobytes.py b/mediagrains/asyncio/aiobytes.py similarity index 96% rename from mediagrains_py36/asyncio/aiobytes.py rename to mediagrains/asyncio/aiobytes.py index 8d1c237..2481acb 100644 --- a/mediagrains_py36/asyncio/aiobytes.py +++ b/mediagrains/asyncio/aiobytes.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2019 British Broadcasting Corporation # @@ -20,6 +19,7 @@ """ from collections.abc import Sequence +from typing import List __all__ = ["AsyncIOBytes"] @@ -40,7 +40,7 @@ class AsyncLazyLoader (object): Unlike the synchronous version loading is not automatic, but can be triggered by awaiting the load coroutine. """ - _attributes = [] + _attributes: List[str] = [] def __init__(self, loader): """ @@ -50,7 +50,7 @@ def __init__(self, loader): self._loader = loader def __getattribute__(self, attr): - if attr in (['_object', '_loader', '__repr__', 'load'] + type(self)._attributes): + if attr in (['_object', '_loader', '__repr__', 'load', '__class__'] + type(self)._attributes): return object.__getattribute__(self, attr) else: if object.__getattribute__(self, '_object') is None: diff --git a/mediagrains_py36/asyncio/bytesaio.py b/mediagrains/asyncio/bytesaio.py similarity index 99% rename from mediagrains_py36/asyncio/bytesaio.py rename to mediagrains/asyncio/bytesaio.py index 7786e8e..9d9247e 100644 --- a/mediagrains_py36/asyncio/bytesaio.py +++ b/mediagrains/asyncio/bytesaio.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2019 British Broadcasting Corporation # diff --git a/mediagrains_py36/__init__.py b/mediagrains/asyncio/py.typed similarity index 100% rename from mediagrains_py36/__init__.py rename to mediagrains/asyncio/py.typed diff --git a/mediagrains/cogenums.py b/mediagrains/cogenums.py index 7b52cec..d46952c 100644 --- a/mediagrains/cogenums.py +++ b/mediagrains/cogenums.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -23,7 +22,7 @@ # that python code using it is compatible with this library when specifying # video and audio formats. -from enum import IntEnum, Enum +from enum import IntEnum __all__ = [ 'CogFrameFormat', @@ -129,10 +128,10 @@ class PlanarChromaFormat(IntEnum): YUV_444 = 0x00 YUV_422 = 0x01 YUV_420 = 0x03 - RGB = 0x10 + RGB = 0x10 -def COG_PLANAR_FORMAT(chroma, depth): +def COG_PLANAR_FORMAT(chroma: PlanarChromaFormat, depth: int) -> CogFrameFormat: if depth <= 8: return CogFrameFormat(0 + chroma + (depth << 10)) elif depth > 16: @@ -141,27 +140,27 @@ def COG_PLANAR_FORMAT(chroma, depth): return CogFrameFormat(4 + chroma + (depth << 10)) -def COG_FRAME_IS_PACKED(fmt): +def COG_FRAME_IS_PACKED(fmt: CogFrameFormat) -> bool: return ((fmt >> 8) & 0x1) != 0 -def COG_FRAME_IS_COMPRESSED(fmt): +def COG_FRAME_IS_COMPRESSED(fmt: CogFrameFormat) -> bool: return ((fmt >> 9) & 0x1) != 0 -def COG_FRAME_IS_PLANAR(fmt): +def COG_FRAME_IS_PLANAR(fmt: CogFrameFormat) -> bool: return ((fmt >> 8) & 0x3) == 0 -def COG_FRAME_IS_ALPHA(fmt): +def COG_FRAME_IS_ALPHA(fmt: CogFrameFormat) -> bool: return ((fmt >> 7) & 0x1) != 0 -def COG_FRAME_IS_PLANAR_RGB(fmt): +def COG_FRAME_IS_PLANAR_RGB(fmt: CogFrameFormat) -> bool: return ((fmt >> 4) & 0x31) == 1 -def COG_FRAME_FORMAT_BYTES_PER_VALUE(fmt): +def COG_FRAME_FORMAT_BYTES_PER_VALUE(fmt: CogFrameFormat) -> int: if ((fmt) & 0xc) == 0: return 1 elif ((fmt) & 0xc) == 4: @@ -170,13 +169,13 @@ def COG_FRAME_FORMAT_BYTES_PER_VALUE(fmt): return 4 -def COG_FRAME_FORMAT_H_SHIFT(fmt): +def COG_FRAME_FORMAT_H_SHIFT(fmt: CogFrameFormat) -> int: return (fmt & 0x1) -def COG_FRAME_FORMAT_V_SHIFT(fmt): +def COG_FRAME_FORMAT_V_SHIFT(fmt: CogFrameFormat) -> int: return ((fmt >> 1) & 0x1) -def COG_FRAME_FORMAT_ACTIVE_BITS(fmt): +def COG_FRAME_FORMAT_ACTIVE_BITS(fmt: CogFrameFormat) -> int: return (((int(fmt)) >> 10) & 0x3F) diff --git a/mediagrains/comparison/__init__.py b/mediagrains/comparison/__init__.py index ac3f15e..07e6791 100644 --- a/mediagrains/comparison/__init__.py +++ b/mediagrains/comparison/__init__.py @@ -25,9 +25,6 @@ The main interface is via the compare_grain function. """ -from __future__ import print_function -from __future__ import absolute_import - from ._internal import GrainComparisonResult, GrainIteratorComparisonResult from .psnr import compute_psnr diff --git a/mediagrains/comparison/__main__.py b/mediagrains/comparison/__main__.py index b22174b..05d1417 100644 --- a/mediagrains/comparison/__main__.py +++ b/mediagrains/comparison/__main__.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Copyright 2018 British Broadcasting Corporation # @@ -15,16 +15,13 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - from fractions import Fraction from . import compare_grain from .options import Exclude -from .testsignalgenerator import LumaSteps +from ..testsignalgenerator import LumaSteps from uuid import uuid1 src_id = uuid1() diff --git a/mediagrains/comparison/_internal.py b/mediagrains/comparison/_internal.py index 018ba38..becf904 100644 --- a/mediagrains/comparison/_internal.py +++ b/mediagrains/comparison/_internal.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -15,13 +14,10 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - from mediatimestamp.immutable import TimeOffset from difflib import SequenceMatcher -from six.moves import reduce +from functools import reduce import struct import sys diff --git a/mediagrains/comparison/options.py b/mediagrains/comparison/options.py index c0ff9be..9dfcaa2 100644 --- a/mediagrains/comparison/options.py +++ b/mediagrains/comparison/options.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -40,10 +39,6 @@ CompareOnlyMetadata is a convenience name for Exclude.data""" -from __future__ import print_function -from __future__ import absolute_import - - __all__ = ["Exclude", "Include", "ExpectedDifference", "CompareOnlyMetadata", "PSNR"] diff --git a/mediagrains/comparison/psnr.py b/mediagrains/comparison/psnr.py index ae633e8..e3f53ec 100644 --- a/mediagrains/comparison/psnr.py +++ b/mediagrains/comparison/psnr.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,102 +13,63 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - -from sys import version_info - -if version_info[0] > 3 or (version_info[0] == 3 and version_info[1] >= 6): - from mediagrains_py36.psnr import compute_psnr - - __all__ = ["compute_psnr"] - -else: - import math - import numpy as np - - from ..cogenums import COG_FRAME_FORMAT_BYTES_PER_VALUE, COG_FRAME_FORMAT_ACTIVE_BITS - from ..cogenums import COG_FRAME_IS_COMPRESSED, COG_FRAME_IS_PACKED - - __all__ = ["compute_psnr"] - - - def _compute_comp_mse(format, data_a, comp_a, data_b, comp_b): - """Compute MSE (Mean Squared Error) for video component. - - Currently supports planar components only. - - :param format: The COG format - :param data_a: Data bytes for GRAIN component a - :param comp_a: COMPONENT for GRAIN a - :param data_b: Data bytes for GRAIN component b - :param comp_b: COMPONENT for GRAIN b - :returns: The MSE value - """ - if COG_FRAME_IS_PACKED(format): - raise NotImplementedError("Packed video format is not supported in this version of python") - - bpp = COG_FRAME_FORMAT_BYTES_PER_VALUE(format) - if bpp == 1: - dtype = np.uint8 - elif bpp == 2: - dtype = np.uint16 - elif bpp == 4: - dtype = np.uint32 - - total = 0 - for y in range(0, comp_a.height): - line_a = data_a[y*comp_a.stride + comp_a.offset:y*comp_a.stride + comp_a.offset + comp_a.width*bpp] - line_b = data_b[y*comp_b.stride + comp_b.offset:y*comp_b.stride + comp_b.offset + comp_b.width*bpp] - np_line_a = np.frombuffer(line_a, dtype=dtype) - np_line_b = np.frombuffer(line_b, dtype=dtype) - total += np.sum(np.square(np.subtract(np_line_a, np_line_b))) - - return total / (comp_a.width*comp_a.height) - - - def _compute_comp_psnr(format, data_a, comp_a, data_b, comp_b, max_val): - """Compute PSNR for video component. - - Currently supports planar components only. - - :param format: The COG format - :param data_a: Data bytes for GRAIN component a - :param comp_a: COMPONENT for GRAIN a - :param data_b: Data bytes for GRAIN component b - :param comp_b: COMPONENT for GRAIN b - :param max_val: Maximum value for a component pixel - :returns: The PSNR - """ - mse = _compute_comp_mse(format, data_a, comp_a, data_b, comp_b) - if mse == 0: - return float('Inf') - else: - return 10.0 * math.log10((max_val**2)/mse) - - - def compute_psnr(grain_a, grain_b): - """Compute PSNR for video grains. - - :param grain_a: A video GRAIN - :param grain_b: A video GRAIN - :returns: A list of PSNR value for each video component - """ - if grain_a.grain_type != grain_b.grain_type or grain_a.grain_type != "video": - raise AttributeError("Invalid grain types") - if grain_a.width != grain_b.width or grain_a.height != grain_b.height: - raise AttributeError("Frame dimensions differ") - - if grain_a.format != grain_b.format: - raise NotImplementedError("Different grain formats not supported") - if COG_FRAME_IS_COMPRESSED(grain_a.format): - raise NotImplementedError("Compressed video is not supported") - - psnr = [] - data_a = bytes(grain_a.data) - data_b = bytes(grain_b.data) - max_val = (1 << COG_FRAME_FORMAT_ACTIVE_BITS(grain_a.format)) - 1 - for comp_a, comp_b in zip(grain_a.components, grain_b.components): - psnr.append(_compute_comp_psnr(grain_a.format, data_a, comp_a, data_b, comp_b, max_val)) - - return psnr +import math +import numpy as np + +from mediagrains.cogenums import COG_FRAME_IS_COMPRESSED, COG_FRAME_FORMAT_ACTIVE_BITS +from mediagrains.numpy import VideoGrain as numpy_VideoGrain, VIDEOGRAIN as numpy_VIDEOGRAIN + +__all__ = ["compute_psnr"] + + +def _compute_comp_mse(data_a, data_b): + """Compute MSE (Mean Squared Error) for video component. + + :param data_a: Data for component a + :param data_b: Data for component b + :returns: The MSE value + """ + return np.mean(np.square(np.subtract(data_a, data_b))) + + +def _compute_comp_psnr(data_a, data_b, max_val): + """Compute PSNR for video component. + + :param data_a: Data for component a + :param data_b: Data for component b + :param max_val: Maximum value for a component pixel + :returns: The PSNR + """ + mse = _compute_comp_mse(data_a, data_b) + if mse == 0: + return float('Inf') + else: + return 10.0 * math.log10((max_val**2)/mse) + + +def compute_psnr(grain_a, grain_b): + """Compute PSNR for video grains. + + :param grain_a: A VIDEOGRAIN + :param grain_b: A VIDEOGRAIN + :returns: A list of PSNR value for each video component + """ + if grain_a.grain_type != grain_b.grain_type or grain_a.grain_type != "video": + raise AttributeError("Invalid grain types") + if grain_a.width != grain_b.width or grain_a.height != grain_b.height: + raise AttributeError("Frame dimensions differ") + + if COG_FRAME_IS_COMPRESSED(grain_a.format): + raise NotImplementedError("Compressed video is not supported") + + if not isinstance(grain_a, numpy_VIDEOGRAIN): + grain_a = numpy_VideoGrain(grain_a) + if not isinstance(grain_b, numpy_VIDEOGRAIN): + grain_b = numpy_VideoGrain(grain_b) + + psnr = [] + max_val = (1 << COG_FRAME_FORMAT_ACTIVE_BITS(grain_a.format)) - 1 + for comp_data_a, comp_data_b in zip(grain_a.component_data, grain_b.component_data): + psnr.append(_compute_comp_psnr(comp_data_a, comp_data_b, max_val)) + + return psnr diff --git a/mediagrains/comparison/py.typed b/mediagrains/comparison/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains/grain.py b/mediagrains/grain.py index f3363dc..a9adad3 100644 --- a/mediagrains/grain.py +++ b/mediagrains/grain.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -21,16 +20,44 @@ directly by client code, but their documentation may be instructive. """ -from __future__ import print_function -from __future__ import absolute_import - -from six import string_types - from uuid import UUID from mediatimestamp.immutable import Timestamp, TimeOffset, TimeRange -from collections import Sequence, MutableSequence, Mapping +from collections.abc import Sequence, MutableSequence, Mapping from fractions import Fraction from copy import copy, deepcopy +from inspect import isawaitable + +from typing import ( + List, + Dict, + Any, + Union, + SupportsBytes, + Optional, + overload, + Tuple, + cast, + Sized, + Iterator, + Iterable, + Awaitable) +from typing_extensions import Literal +from .typing import ( + RationalTypes, + MediaJSONSerialisable, + EventGrainDatumDict, + GrainMetadataDict, + GrainDataType, + VideoGrainComponentDict, + EmptyGrainMetadataDict, + FractionDict, + TimeLabel, + EventGrainMetadataDict, + VideoGrainMetadataDict, + CodedVideoGrainMetadataDict, + AudioGrainMetadataDict, + CodedAudioGrainMetadataDict, + GrainDataParameterType) from .cogenums import CogFrameFormat, CogFrameLayout, CogAudioFormat @@ -39,8 +66,8 @@ __all__ = ["GRAIN", "VIDEOGRAIN", "AUDIOGRAIN", "CODEDVIDEOGRAIN", "CODEDAUDIOGRAIN", "EVENTGRAIN", "attributes_for_grain_type"] -def attributes_for_grain_type(grain_type): - """Returns a list of attributes for a particular grain type. Useful for testing.""" +def attributes_for_grain_type(grain_type: str) -> List[str]: + """Returns a list of attributes for a partiggcular grain type. Useful for testing.""" COMMON_ATTRS = ['source_id', 'flow_id', 'origin_timestamp', 'sync_timestamp', 'creation_timestamp', 'rate', 'duration'] @@ -67,8 +94,10 @@ class GRAIN(Sequence): (meta, data) -where meta is a dictionary containing the grain metadata, and data is a python -buffer object representing the payload (or None for an empty grain). +where meta is a dictionary containing the grain metadata, and data is None or one of the following: +* a bytes-like object +* An object supporting the __bytes__ magic method +* An awaitable returning a valid data element In addition the class provides a number of properties which can be used to access parts of the standard grain metadata, and all other grain classes @@ -78,10 +107,17 @@ class GRAIN(Sequence): The meta dictionary object data - Either None or an object which can be cast to bytes by passing it to the bytes - constructor and will in of itself respond to the python-level portions of the bytes-like - object protocol. It is not guaranteed that this object will always respond correctly to the - C buffer-protocol, but it can always be converted into something that will by calling bytes on it. + One of the following: + * A byteslike object -- This becomes the grain's data element + * An object that has a method __bytes__ which returns a bytes-like object, which will be the grain's data element + * None -- This grain has no data + + If the data parameter passed on construction is an awaitable which will return a valid data element when awaited then the grain's data element is + initially None, but the grain can be awaited to populate it + + For convenience any grain can be awaited and will return the data element, regardless of whether the underlying data is asynchronous or not + + For additional convenience using a grain as an async context manager will ensure that the data element is populated if it needs to be and can be. grain_type A string containing the type of the grain, any value is possible @@ -134,192 +170,237 @@ class GRAIN(Sequence): Returns a normalised Timestamp, TimeOffset or TimeRange using the video frame rate or audio sample rate. """ - def __init__(self, meta, data): + def __init__(self, meta: GrainMetadataDict, data: GrainDataParameterType): self.meta = meta - self._data = data + + self._data_fetcher_coroutine: Optional[Awaitable[Optional[GrainDataType]]] + self._data_fetcher_length: int = 0 + self._data: Optional[GrainDataType] + + if isawaitable(data): + self._data_fetcher_coroutine = cast(Awaitable[Optional[GrainDataType]], data) + self._data = None + else: + self._data_fetcher_coroutine = None + self._data = cast(Optional[GrainDataType], data) self._factory = "Grain" + + # This code is here to deal with malformed inputs, and as such needs to cast away the type safety to operate if "@_ns" not in self.meta: - self.meta['@_ns'] = "urn:x-ipstudio:ns:0.1" + cast(EmptyGrainMetadataDict, self.meta)['@_ns'] = "urn:x-ipstudio:ns:0.1" if 'grain' not in self.meta: - self.meta['grain'] = {} + cast(dict, self.meta)['grain'] = {} if 'grain_type' not in self.meta['grain']: - self.meta['grain']['grain_type'] = "empty" + cast(EmptyGrainMetadataDict, self.meta)['grain']['grain_type'] = "empty" if 'creation_timestamp' not in self.meta['grain']: - self.meta['grain']['creation_timestamp'] = str(Timestamp.get_time()) + cast(EmptyGrainMetadataDict, self.meta)['grain']['creation_timestamp'] = str(Timestamp.get_time()) if 'origin_timestamp' not in self.meta['grain']: - self.meta['grain']['origin_timestamp'] = self.meta['grain']['creation_timestamp'] + cast(EmptyGrainMetadataDict, self.meta)['grain']['origin_timestamp'] = self.meta['grain']['creation_timestamp'] if 'sync_timestamp' not in self.meta['grain']: - self.meta['grain']['sync_timestamp'] = self.meta['grain']['origin_timestamp'] + cast(EmptyGrainMetadataDict, self.meta)['grain']['sync_timestamp'] = self.meta['grain']['origin_timestamp'] if 'rate' not in self.meta['grain']: - self.meta['grain']['rate'] = {'numerator': 0, - 'denominator': 1} + cast(EmptyGrainMetadataDict, self.meta)['grain']['rate'] = {'numerator': 0, + 'denominator': 1} if 'duration' not in self.meta['grain']: - self.meta['grain']['duration'] = {'numerator': 0, - 'denominator': 1} + cast(EmptyGrainMetadataDict, self.meta)['grain']['duration'] = {'numerator': 0, + 'denominator': 1} if 'source_id' not in self.meta['grain']: - self.meta['grain']['source_id'] = "00000000-0000-0000-0000-000000000000" + cast(EmptyGrainMetadataDict, self.meta)['grain']['source_id'] = "00000000-0000-0000-0000-000000000000" if 'flow_id' not in self.meta['grain']: - self.meta['grain']['flow_id'] = "00000000-0000-0000-0000-000000000000" + cast(EmptyGrainMetadataDict, self.meta)['grain']['flow_id'] = "00000000-0000-0000-0000-000000000000" if isinstance(self.meta["grain"]["source_id"], UUID): - self.meta['grain']['source_id'] = str(self.meta['grain']['source_id']) + cast(EmptyGrainMetadataDict, self.meta)['grain']['source_id'] = str(self.meta['grain']['source_id']) if isinstance(self.meta["grain"]["flow_id"], UUID): - self.meta['grain']['flow_id'] = str(self.meta['grain']['flow_id']) + cast(EmptyGrainMetadataDict, self.meta)['grain']['flow_id'] = str(self.meta['grain']['flow_id']) if isinstance(self.meta["grain"]["origin_timestamp"], Timestamp): - self.meta['grain']['origin_timestamp'] = str(self.meta['grain']['origin_timestamp']) + cast(EmptyGrainMetadataDict, self.meta)['grain']['origin_timestamp'] = str(self.meta['grain']['origin_timestamp']) if isinstance(self.meta["grain"]["sync_timestamp"], Timestamp): - self.meta['grain']['sync_timestamp'] = str(self.meta['grain']['sync_timestamp']) + cast(EmptyGrainMetadataDict, self.meta)['grain']['sync_timestamp'] = str(self.meta['grain']['sync_timestamp']) if isinstance(self.meta["grain"]["creation_timestamp"], Timestamp): - self.meta['grain']['creation_timestamp'] = str(self.meta['grain']['creation_timestamp']) + cast(EmptyGrainMetadataDict, self.meta)['grain']['creation_timestamp'] = str(self.meta['grain']['creation_timestamp']) if isinstance(self.meta['grain']['rate'], Fraction): - self.meta['grain']['rate'] = {'numerator': self.meta['grain']['rate'].numerator, - 'denominator': self.meta['grain']['rate'].denominator} + cast(EmptyGrainMetadataDict, self.meta)['grain']['rate'] = {'numerator': self.meta['grain']['rate'].numerator, + 'denominator': self.meta['grain']['rate'].denominator} if isinstance(self.meta['grain']['duration'], Fraction): - self.meta['grain']['duration'] = {'numerator': self.meta['grain']['duration'].numerator, - 'denominator': self.meta['grain']['duration'].denominator} + cast(EmptyGrainMetadataDict, self.meta)['grain']['duration'] = {'numerator': self.meta['grain']['duration'].numerator, + 'denominator': self.meta['grain']['duration'].denominator} - def __len__(self): + def __len__(self) -> int: return 2 - def __getitem__(self, index): - if index == 0: - return self.meta - elif index == 1: - return self.data - else: - raise IndexError("tuple index out of range") + @overload + def __getitem__(self, index: int) -> Union[GrainMetadataDict, Optional[GrainDataType]]: ... + + @overload # noqa: F811 + def __getitem__(self, index: slice) -> Union[Tuple[GrainMetadataDict], + Tuple[GrainMetadataDict, Optional[GrainDataType]], + Tuple[Optional[GrainDataType]], + Tuple[()]]: ... - def __repr__(self): - if self.data is None: + def __getitem__(self, index): # noqa: F811 + return (self.meta, self.data)[index] + + def __repr__(self) -> str: + if not hasattr(self.data, "__len__"): return "{}({!r})".format(self._factory, self.meta) else: - return "{}({!r},< binary data of length {} >)".format(self._factory, self.meta, len(self.data)) + return "{}({!r},< binary data of length {} >)".format(self._factory, self.meta, len(cast(Sized, self.data))) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return tuple(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) - def __copy__(self): + def __copy__(self) -> "GRAIN": from .grain_constructors import Grain return Grain(copy(self.meta), self.data) - def __deepcopy__(self, memo): + def __deepcopy__(self, memo) -> "GRAIN": from .grain_constructors import Grain return Grain(deepcopy(self.meta), deepcopy(self.data)) - def __bytes__(self): + def __bytes__(self) -> Optional[bytes]: + if isinstance(self._data, bytes): + return self._data + elif self._data is None: + return None return bytes(self._data) + def has_data(self) -> bool: + return self._data is not None + + async def __await__(self) -> Optional[GrainDataType]: + if self._data is None and self._data_fetcher_coroutine is not None: + self._data = await self._data_fetcher_coroutine + return self._data + + async def __aenter__(self): + await self + return self + + async def __aexit__(self, *args, **kwargs): + pass + @property - def data(self): + def data(self) -> Optional[GrainDataType]: return self._data @data.setter - def data(self, value): - self._data = value + def data(self, value: GrainDataParameterType): + if isawaitable(value): + self._data = None + self._data_fetcher_coroutine = cast(Awaitable[Optional[GrainDataType]], value) + else: + self._data = cast(Optional[GrainDataType], value) + self._data_fetcher_coroutine = None @property - def grain_type(self): + def grain_type(self) -> str: return self.meta['grain']['grain_type'] @grain_type.setter - def grain_type(self, value): - self.meta['grain']['grain_type'] = value + def grain_type(self, value: str) -> None: + # We ignore the type safety rules for this assignment + self.meta['grain']['grain_type'] = value # type: ignore @property - def source_id(self): - return UUID(self.meta['grain']['source_id']) + def source_id(self) -> UUID: + # Our code ensures that this will always be a string at runtime + return UUID(cast(str, self.meta['grain']['source_id'])) @source_id.setter - def source_id(self, value): - self.meta['grain']['source_id'] = str(value) + def source_id(self, value: Union[UUID, str]) -> None: + cast(EmptyGrainMetadataDict, self.meta)['grain']['source_id'] = str(value) @property - def flow_id(self): - return UUID(self.meta['grain']['flow_id']) + def flow_id(self) -> UUID: + return UUID(cast(str, self.meta['grain']['flow_id'])) @flow_id.setter - def flow_id(self, value): - self.meta['grain']['flow_id'] = str(value) + def flow_id(self, value: Union[UUID, str]) -> None: + cast(EmptyGrainMetadataDict, self.meta)['grain']['flow_id'] = str(value) @property - def origin_timestamp(self): - return Timestamp.from_tai_sec_nsec(self.meta['grain']['origin_timestamp']) + def origin_timestamp(self) -> Timestamp: + return Timestamp.from_tai_sec_nsec(cast(str, self.meta['grain']['origin_timestamp'])) @origin_timestamp.setter - def origin_timestamp(self, value): + def origin_timestamp(self, value: Union[TimeOffset, str]): if isinstance(value, TimeOffset): - value = value.to_sec_nsec() - self.meta['grain']['origin_timestamp'] = value + cast(EmptyGrainMetadataDict, self.meta)['grain']['origin_timestamp'] = value.to_sec_nsec() + else: + cast(EmptyGrainMetadataDict, self.meta)['grain']['origin_timestamp'] = value - def final_origin_timestamp(self): + def final_origin_timestamp(self) -> Timestamp: return self.origin_timestamp - def origin_timerange(self): + def origin_timerange(self) -> TimeRange: return TimeRange(self.origin_timestamp, self.final_origin_timestamp(), TimeRange.INCLUSIVE) - def normalise_time(self, value): + def normalise_time(self, value: Timestamp) -> Timestamp: return value @property - def sync_timestamp(self): + def sync_timestamp(self) -> Timestamp: return Timestamp.from_tai_sec_nsec(self.meta['grain']['sync_timestamp']) @sync_timestamp.setter - def sync_timestamp(self, value): + def sync_timestamp(self, value: Union[TimeOffset, str]) -> None: if isinstance(value, TimeOffset): - value = value.to_sec_nsec() - self.meta['grain']['sync_timestamp'] = value + cast(EmptyGrainMetadataDict, self.meta)['grain']['sync_timestamp'] = value.to_sec_nsec() + else: + cast(EmptyGrainMetadataDict, self.meta)['grain']['sync_timestamp'] = value @property - def creation_timestamp(self): + def creation_timestamp(self) -> Timestamp: return Timestamp.from_tai_sec_nsec(self.meta['grain']['creation_timestamp']) @creation_timestamp.setter - def creation_timestamp(self, value): + def creation_timestamp(self, value: Union[TimeOffset, str]) -> None: if isinstance(value, TimeOffset): - value = value.to_sec_nsec() - self.meta['grain']['creation_timestamp'] = value + cast(EmptyGrainMetadataDict, self.meta)['grain']['creation_timestamp'] = value.to_sec_nsec() + else: + cast(EmptyGrainMetadataDict, self.meta)['grain']['creation_timestamp'] = value @property - def rate(self): - return Fraction(self.meta['grain']['rate']['numerator'], - self.meta['grain']['rate']['denominator']) + def rate(self) -> Fraction: + return Fraction(cast(FractionDict, self.meta['grain']['rate'])['numerator'], + cast(FractionDict, self.meta['grain']['rate'])['denominator']) @rate.setter - def rate(self, value): + def rate(self, value: RationalTypes) -> None: value = Fraction(value) - self.meta['grain']['rate'] = { + cast(EmptyGrainMetadataDict, self.meta)['grain']['rate'] = { 'numerator': value.numerator, 'denominator': value.denominator - } + } @property - def duration(self): - return Fraction(self.meta['grain']['duration']['numerator'], - self.meta['grain']['duration']['denominator']) + def duration(self) -> Fraction: + return Fraction(cast(FractionDict, self.meta['grain']['duration'])['numerator'], + cast(FractionDict, self.meta['grain']['duration'])['denominator']) @duration.setter - def duration(self, value): + def duration(self, value: RationalTypes) -> None: value = Fraction(value) - self.meta['grain']['duration'] = { + cast(EmptyGrainMetadataDict, self.meta)['grain']['duration'] = { 'numerator': value.numerator, 'denominator': value.denominator - } + } @property - def timelabels(self): + def timelabels(self) -> "GRAIN.TIMELABELS": return GRAIN.TIMELABELS(self) @timelabels.setter - def timelabels(self, value): - self.meta['grain']['timelabels'] = [] + def timelabels(self, value: "Union[List[GRAIN.TIMELABEL], GRAIN.TIMELABELS]") -> None: + cast(EmptyGrainMetadataDict, self.meta)['grain']['timelabels'] = [] for x in value: self.timelabels.append(x) - def add_timelabel(self, tag, count, rate, drop_frame=False): + def add_timelabel(self, tag: str, count: int, rate: Fraction, drop_frame: bool = False) -> None: tl = GRAIN.TIMELABEL() tl.tag = tag tl.count = count @@ -328,7 +409,9 @@ def add_timelabel(self, tag, count, rate, drop_frame=False): self.timelabels.append(tl) class TIMELABEL(Mapping): - def __init__(self, meta=None): + GrainMetadataDict = Dict[str, Any] + + def __init__(self, meta: "Optional[GRAIN.TIMELABEL.GrainMetadataDict]" = None): if meta is None: meta = {} self.meta = meta @@ -345,109 +428,141 @@ def __init__(self, meta=None): if 'drop_frame' not in self.meta['timelabel']: self.meta['timelabel']['drop_frame'] = False - def __getitem__(self, key): + def __getitem__(self, key: str) -> Union[str, Dict[str, Union[int, bool]]]: return self.meta[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Union[str, Dict[str, Union[int, bool]]]) -> None: if key not in ['tag', 'timelabel']: raise KeyError self.meta[key] = value - def __iter__(self): + def __iter__(self) -> Iterator[str]: return self.meta.__iter__() - def __len__(self): + def __len__(self) -> int: return 2 - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return dict(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) @property - def tag(self): + def tag(self) -> str: return self.meta['tag'] @tag.setter - def tag(self, value): + def tag(self, value: str) -> None: self.meta['tag'] = value @property - def count(self): + def count(self) -> int: return self.meta['timelabel']['frames_since_midnight'] @count.setter - def count(self, value): + def count(self, value: int) -> None: self.meta['timelabel']['frames_since_midnight'] = int(value) @property - def rate(self): + def rate(self) -> Fraction: return Fraction(self.meta['timelabel']['frame_rate_numerator'], self.meta['timelabel']['frame_rate_denominator']) @rate.setter - def rate(self, value): + def rate(self, value: RationalTypes) -> None: value = Fraction(value) self.meta['timelabel']['frame_rate_numerator'] = value.numerator self.meta['timelabel']['frame_rate_denominator'] = value.denominator @property - def drop_frame(self): + def drop_frame(self) -> bool: return self.meta['timelabel']['drop_frame'] @drop_frame.setter - def drop_frame(self, value): + def drop_frame(self, value: bool) -> None: self.meta['timelabel']['drop_frame'] = bool(value) class TIMELABELS(MutableSequence): - def __init__(self, parent): + def __init__(self, parent: "GRAIN"): self.parent = parent - def __getitem__(self, key): + @overload + def __getitem__(self, key: int) -> "GRAIN.TIMELABEL": ... + + @overload # noqa: F811 + def __getitem__(self, key: slice) -> "List[GRAIN.TIMELABEL]": ... + + def __getitem__(self, key): # noqa: F811 if 'timelabels' not in self.parent.meta['grain']: raise IndexError("list index out of range") - return GRAIN.TIMELABEL(self.parent.meta['grain']['timelabels'][key]) + if isinstance(key, int): + return GRAIN.TIMELABEL(self.parent.meta['grain']['timelabels'][key]) + else: + return [GRAIN.TIMELABEL(self.parent.meta['grain']['timelabels'][n]) for n in range(len(self))[key]] + + @overload + def __setitem__(self, key: int, value: "GRAIN.TIMELABEL.GrainMetadataDict") -> None: ... + + @overload # noqa: F811 + def __setitem__(self, key: slice, value: "Iterable[GRAIN.TIMELABEL.GrainMetadataDict]") -> None: ... - def __setitem__(self, key, value): + def __setitem__(self, key, value): # noqa: F811 if 'timelabels' not in self.parent.meta['grain']: raise IndexError("list assignment index out of range") - self.parent.meta['grain']['timelabels'][key] = dict(GRAIN.TIMELABEL(value)) + if isinstance(key, int): + self.parent.meta['grain']['timelabels'][key] = dict(GRAIN.TIMELABEL(value)) + else: + values = iter(value) + for n in key: + self.parent.meta['grain']['timelabels'][n] = dict(GRAIN.TIMELABEL(next(values))) - def __delitem__(self, key): + def __delitem__(self, key: Union[int, slice]) -> None: if 'timelabels' not in self.parent.meta['grain']: raise IndexError("list assignment index out of range") + del self.parent.meta['grain']['timelabels'][key] if len(self.parent.meta['grain']['timelabels']) == 0: del self.parent.meta['grain']['timelabels'] - def insert(self, key, value): + def insert(self, key: int, value: "GRAIN.TIMELABEL.GrainMetadataDict") -> None: if 'timelabels' not in self.parent.meta['grain']: - self.parent.meta['grain']['timelabels'] = [] - self.parent.meta['grain']['timelabels'].insert(key, dict(GRAIN.TIMELABEL(value))) + cast(EmptyGrainMetadataDict, self.parent.meta)['grain']['timelabels'] = [] + self.parent.meta['grain']['timelabels'].insert(key, cast(TimeLabel, dict(GRAIN.TIMELABEL(value)))) - def __len__(self): + def __len__(self) -> int: if 'timelabels' not in self.parent.meta['grain']: return 0 return len(self.parent.meta['grain']['timelabels']) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return list(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) @property - def length(self): - if self.data is not None: - return len(self.data) + def length(self) -> int: + if hasattr(self.data, "__len__"): + return len(cast(Sized, self.data)) + elif hasattr(self.data, "__bytes__"): + return len(bytes(cast(SupportsBytes, self.data))) + elif self.data is None and self._data_fetcher_coroutine is not None: + return self._data_fetcher_length else: return 0 + @length.setter + def length(self, L: int) -> None: + if self.data is None and self._data_fetcher_coroutine is not None: + self._data_fetcher_length = L + else: + raise AttributeError + @property - def expected_length(self): + def expected_length(self) -> int: if 'length' in self.meta['grain']: - return self.meta['grain']['length'] + return cast(dict, self.meta['grain'])['length'] else: return self.length @@ -526,14 +641,24 @@ class EVENTGRAIN(GRAIN): provided string, and pre and post set optionally. All calls should use only json serialisable objects for the values of pre and post. """ - def __init__(self, meta, data): + def __init__(self, meta: EventGrainMetadataDict, data: GrainDataParameterType): super(EVENTGRAIN, self).__init__(meta, None) + self.meta: EventGrainMetadataDict + self._factory = "EventGrain" self.meta['grain']['grain_type'] = 'event' if 'event_payload' not in self.meta['grain']: - self.meta['grain']['event_payload'] = {} - if data is not None: - self.data = data + self.meta['grain']['event_payload'] = { + 'type': "", + 'topic': "", + 'data': []} + if isawaitable(data): + self._data_fetcher_coroutine = cast(Awaitable[Optional[GrainDataType]], data) + elif data is not None: + if isinstance(data, bytes): + self.data = data + else: + self.data = bytes(cast(SupportsBytes, data)) if 'type' not in self.meta['grain']['event_payload']: self.meta['grain']['event_payload']['type'] = "" if 'topic' not in self.meta['grain']['event_payload']: @@ -542,46 +667,48 @@ def __init__(self, meta, data): self.meta['grain']['event_payload']['data'] = [] @property - def data(self): + def data(self) -> bytes: return json.dumps({'type': self.event_type, 'topic': self.topic, 'data': [dict(datum) for datum in self.event_data]}).encode('utf-8') @data.setter - def data(self, value): - if not isinstance(value, string_types): - value = value.decode('utf-8') - value = json.loads(value) - if 'type' not in value or 'topic' not in value or 'data' not in value: + def data(self, value: Union[str, bytes]): + if not isinstance(value, str): + payload = json.loads(value.decode('utf-8')) + else: + payload = json.loads(value) + + if 'type' not in payload or 'topic' not in payload or 'data' not in payload: raise ValueError("incorrectly formated event payload") - self.event_type = value['type'] - self.topic = value['topic'] + self.event_type = payload['type'] + self.topic = payload['topic'] self.meta['grain']['event_payload']['data'] = [] - for datum in value['data']: - d = {'path': datum['path']} + for datum in payload['data']: + d: EventGrainDatumDict = {'path': datum['path']} if 'pre' in datum: d['pre'] = datum['pre'] if 'post' in datum: d['post'] = datum['post'] self.meta['grain']['event_payload']['data'].append(d) - def __repr__(self): + def __repr__(self) -> str: return "EventGrain({!r})".format(self.meta) @property - def event_type(self): + def event_type(self) -> str: return self.meta['grain']['event_payload']['type'] @event_type.setter - def event_type(self, value): + def event_type(self, value: str) -> None: self.meta['grain']['event_payload']['type'] = value @property - def topic(self): + def topic(self) -> str: return self.meta['grain']['event_payload']['topic'] @topic.setter - def topic(self, value): + def topic(self, value: str) -> None: self.meta['grain']['event_payload']['topic'] = value class DATA(Mapping): @@ -607,76 +734,76 @@ class DATA(Mapping): The post value, or None if none is present. If set to None will remove "post" key from dictionary. """ - def __init__(self, meta): + def __init__(self, meta: EventGrainDatumDict): self.meta = meta - def __getitem__(self, key): + def __getitem__(self, key: Literal['path', 'pre', 'post']) -> MediaJSONSerialisable: return self.meta[key] - def __setitem__(self, key, value): + def __setitem__(self, key: Literal['path', 'pre', 'post'], value: MediaJSONSerialisable) -> None: self.meta[key] = value - def __delitem__(self, key): + def __delitem__(self, key: Literal['pre', 'post']) -> None: del self.meta[key] - def __iter__(self): + def __iter__(self) -> Iterator[str]: return self.meta.__iter__() - def __len__(self): + def __len__(self) -> int: return self.meta.__len__() - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return dict(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) @property - def path(self): + def path(self) -> str: return self.meta['path'] @path.setter - def path(self, value): + def path(self, value: str) -> None: self.meta['path'] = value @property - def pre(self): + def pre(self) -> Optional[MediaJSONSerialisable]: if 'pre' in self.meta: return self.meta['pre'] else: return None @pre.setter - def pre(self, value): + def pre(self, value: Optional[MediaJSONSerialisable]) -> None: if value is not None: self.meta['pre'] = value else: del self.meta['pre'] @property - def post(self): + def post(self) -> Optional[MediaJSONSerialisable]: if 'post' in self.meta: return self.meta['post'] - elif 'pre' in self.meta: + else: return None @post.setter - def post(self, value): + def post(self, value: Optional[MediaJSONSerialisable]) -> None: if value is not None: self.meta['post'] = value elif 'post' in self.meta: del self.meta['post'] @property - def event_data(self): + def event_data(self) -> List["EVENTGRAIN.DATA"]: return [EVENTGRAIN.DATA(datum) for datum in self.meta['grain']['event_payload']['data']] @event_data.setter - def event_data(self, value): - self.meta['grain']['event_payload']['data'] = [dict(datum) for datum in value] + def event_data(self, value: List[EventGrainDatumDict]) -> None: + self.meta['grain']['event_payload']['data'] = [cast(EventGrainDatumDict, dict(datum)) for datum in value] - def append(self, path, pre=None, post=None): - datum = {'path': path} + def append(self, path: str, pre: Optional[MediaJSONSerialisable] = None, post: Optional[MediaJSONSerialisable] = None) -> None: + datum = EventGrainDatumDict(path=path) if pre is not None: datum['pre'] = pre if post is not None: @@ -796,97 +923,116 @@ class COMPONENT(Mapping): length The total length of the data for this component in bytes """ - def __init__(self, meta): + def __init__(self, meta: VideoGrainComponentDict): self.meta = meta - def __getitem__(self, key): + def __getitem__(self, key: Literal['stride', 'offset', 'width', 'height', 'length']) -> int: return self.meta[key] - def __setitem__(self, key, value): + def __setitem__(self, key: Literal['stride', 'offset', 'width', 'height', 'length'], value: int) -> None: self.meta[key] = value - def __delitem__(self, key): - del self.meta[key] - - def __iter__(self): + def __iter__(self) -> Iterator[str]: return self.meta.__iter__() - def __len__(self): + def __len__(self) -> int: return self.meta.__len__() - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return dict(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) @property - def stride(self): + def stride(self) -> int: return self.meta['stride'] @stride.setter - def stride(self, value): + def stride(self, value: int) -> None: self.meta['stride'] = value @property - def offset(self): + def offset(self) -> int: return self.meta['offset'] @offset.setter - def offset(self, value): + def offset(self, value: int) -> None: self.meta['offset'] = value @property - def width(self): + def width(self) -> int: return self.meta['width'] @width.setter - def width(self, value): + def width(self, value: int) -> None: self.meta['width'] = value @property - def height(self): + def height(self) -> int: return self.meta['height'] @height.setter - def height(self, value): + def height(self, value: int) -> None: self.meta['height'] = value @property - def length(self): + def length(self) -> int: return self.meta['length'] @length.setter - def length(self, value): + def length(self, value: int) -> None: self.meta['length'] = value class COMPONENT_LIST(MutableSequence): - def __init__(self, parent): + def __init__(self, parent: "VIDEOGRAIN"): self.parent = parent - def __getitem__(self, key): - return type(self.parent).COMPONENT(self.parent.meta['grain']['cog_frame']['components'][key]) + @overload + def __getitem__(self, key: int) -> "VIDEOGRAIN.COMPONENT": ... - def __setitem__(self, key, value): - self.parent.meta['grain']['cog_frame']['components'][key] = type(self.parent).COMPONENT(value) + @overload # noqa: F811 + def __getitem__(self, key: slice) -> "List[VIDEOGRAIN.COMPONENT]": ... - def __delitem__(self, key): + def __getitem__(self, key): # noqa: F811 + if isinstance(key, int): + return type(self.parent).COMPONENT(self.parent.meta['grain']['cog_frame']['components'][key]) + else: + return [type(self.parent).COMPONENT(self.parent.meta['grain']['cog_frame']['components'][k]) for k in range(len(self))[key]] + + @overload + def __setitem__(self, key: int, value: VideoGrainComponentDict) -> None: ... + + @overload # noqa: F811 + def __setitem__(self, key: slice, value: Iterable[VideoGrainComponentDict]) -> None: ... + + def __setitem__(self, key, value): # noqa: F811 + if isinstance(key, int): + self.parent.meta['grain']['cog_frame']['components'][key] = type(self.parent).COMPONENT(value) + else: + values = iter(value) + for n in range(len(self))[key]: + self.parent.meta['grain']['cog_frame']['components'][n] = type(self.parent).COMPONENT(next(values)) + + def __delitem__(self, key: Union[int, slice]) -> None: del self.parent.meta['grain']['cog_frame']['components'][key] - def insert(self, key, value): - self.parent.meta['grain']['cog_frame']['components'].insert(key, type(self.parent).COMPONENT(value)) + def insert(self, key: int, value: VideoGrainComponentDict) -> None: + self.parent.meta['grain']['cog_frame']['components'].insert(key, type(self.parent).COMPONENT(value)) # type: ignore - def __len__(self): + def __len__(self) -> int: return len(self.parent.meta['grain']['cog_frame']['components']) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return list(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) - def __init__(self, meta, data): + def __init__(self, meta: VideoGrainMetadataDict, data: GrainDataParameterType): super(VIDEOGRAIN, self).__init__(meta, data) + self.meta: VideoGrainMetadataDict + self._factory = "VideoGrain" self.meta['grain']['grain_type'] = 'video' if 'cog_frame' not in self.meta['grain']: @@ -902,81 +1048,81 @@ def __init__(self, meta, data): self.meta['grain']['cog_frame']['layout'] = int(self.meta['grain']['cog_frame']['layout']) self.components = VIDEOGRAIN.COMPONENT_LIST(self) - def normalise_time(self, value): + def normalise_time(self, value: Timestamp) -> Timestamp: if self.rate == 0: return value return value.normalise(self.rate.numerator, self.rate.denominator) @property - def format(self): + def format(self) -> CogFrameFormat: return CogFrameFormat(self.meta['grain']['cog_frame']['format']) @format.setter - def format(self, value): + def format(self, value: CogFrameFormat) -> None: self.meta['grain']['cog_frame']['format'] = int(value) @property - def width(self): + def width(self) -> int: return self.meta['grain']['cog_frame']['width'] @width.setter - def width(self, value): + def width(self, value: int) -> None: self.meta['grain']['cog_frame']['width'] = value @property - def height(self): + def height(self) -> int: return self.meta['grain']['cog_frame']['height'] @height.setter - def height(self, value): + def height(self, value: int) -> None: self.meta['grain']['cog_frame']['height'] = value @property - def layout(self): + def layout(self) -> CogFrameLayout: return CogFrameLayout(self.meta['grain']['cog_frame']['layout']) @layout.setter - def layout(self, value): + def layout(self, value: CogFrameLayout) -> None: self.meta['grain']['cog_frame']['layout'] = int(value) @property - def extension(self): + def extension(self) -> int: return self.meta['grain']['cog_frame']['extension'] @extension.setter - def extension(self, value): + def extension(self, value: int) -> None: self.meta['grain']['cog_frame']['extension'] = value @property - def source_aspect_ratio(self): + def source_aspect_ratio(self) -> Optional[Fraction]: if 'source_aspect_ratio' in self.meta['grain']['cog_frame']: - return Fraction(self.meta['grain']['cog_frame']['source_aspect_ratio']['numerator'], - self.meta['grain']['cog_frame']['source_aspect_ratio']['denominator']) + return Fraction(cast(FractionDict, self.meta['grain']['cog_frame']['source_aspect_ratio'])['numerator'], + cast(FractionDict, self.meta['grain']['cog_frame']['source_aspect_ratio'])['denominator']) else: return None @source_aspect_ratio.setter - def source_aspect_ratio(self, value): + def source_aspect_ratio(self, value: RationalTypes) -> None: value = Fraction(value) self.meta['grain']['cog_frame']['source_aspect_ratio'] = {'numerator': value.numerator, 'denominator': value.denominator} @property - def pixel_aspect_ratio(self): + def pixel_aspect_ratio(self) -> Optional[Fraction]: if 'pixel_aspect_ratio' in self.meta['grain']['cog_frame']: - return Fraction(self.meta['grain']['cog_frame']['pixel_aspect_ratio']['numerator'], - self.meta['grain']['cog_frame']['pixel_aspect_ratio']['denominator']) + return Fraction(cast(FractionDict, self.meta['grain']['cog_frame']['pixel_aspect_ratio'])['numerator'], + cast(FractionDict, self.meta['grain']['cog_frame']['pixel_aspect_ratio'])['denominator']) else: return None @pixel_aspect_ratio.setter - def pixel_aspect_ratio(self, value): + def pixel_aspect_ratio(self, value: RationalTypes) -> None: value = Fraction(value) self.meta['grain']['cog_frame']['pixel_aspect_ratio'] = {'numerator': value.numerator, 'denominator': value.denominator} @property - def expected_length(self): + def expected_length(self) -> int: length = 0 for component in self.components: if component.offset + component.length > length: @@ -1067,156 +1213,179 @@ class CODEDVIDEOGRAIN(GRAIN): A list-like object containing integer offsets of coded units within the data array. """ - def __init__(self, meta, data): + def __init__(self, meta: CodedVideoGrainMetadataDict, data: GrainDataParameterType): super(CODEDVIDEOGRAIN, self).__init__(meta, data) + self.meta: CodedVideoGrainMetadataDict + self._factory = "CodedVideoGrain" self.meta['grain']['grain_type'] = 'coded_video' if 'cog_coded_frame' not in self.meta['grain']: - self.meta['grain']['cog_coded_frame'] = {} + self.meta['grain']['cog_coded_frame'] = {} # type: ignore if 'format' not in self.meta['grain']['cog_coded_frame']: self.meta['grain']['cog_coded_frame']['format'] = int(CogFrameFormat.UNKNOWN) if 'layout' not in self.meta['grain']['cog_coded_frame']: self.meta['grain']['cog_coded_frame']['layout'] = int(CogFrameLayout.UNKNOWN) - for key in ['origin_width', 'origin_height', 'coded_width', 'coded_height', 'temporal_offset', 'length']: - if key not in self.meta['grain']['cog_coded_frame']: - self.meta['grain']['cog_coded_frame'][key] = 0 + if 'origin_width' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['origin_width'] = 0 + if 'origin_height' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['origin_height'] = 0 + if 'coded_width' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['coded_width'] = 0 + if 'coded_height' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['coded_height'] = 0 + if 'temporal_offset' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['temporal_offset'] = 0 + if 'length' not in self.meta['grain']['cog_coded_frame']: + self.meta['grain']['cog_coded_frame']['length'] = 0 if 'is_key_frame' not in self.meta['grain']['cog_coded_frame']: self.meta['grain']['cog_coded_frame']['is_key_frame'] = False self.meta['grain']['cog_coded_frame']['format'] = int(self.meta['grain']['cog_coded_frame']['format']) self.meta['grain']['cog_coded_frame']['layout'] = int(self.meta['grain']['cog_coded_frame']['layout']) - def normalise_time(self, value): + def normalise_time(self, value: Timestamp) -> Timestamp: if self.rate == 0: return value return value.normalise(self.rate.numerator, self.rate.denominator) @property - def format(self): + def format(self) -> CogFrameFormat: return CogFrameFormat(self.meta['grain']['cog_coded_frame']['format']) @format.setter - def format(self, value): + def format(self, value: CogFrameFormat) -> None: self.meta['grain']['cog_coded_frame']['format'] = int(value) @property - def layout(self): + def layout(self) -> CogFrameLayout: return CogFrameLayout(self.meta['grain']['cog_coded_frame']['layout']) @layout.setter - def layout(self, value): + def layout(self, value: CogFrameLayout) -> None: self.meta['grain']['cog_coded_frame']['layout'] = int(value) @property - def origin_width(self): + def origin_width(self) -> int: return self.meta['grain']['cog_coded_frame']['origin_width'] @origin_width.setter - def origin_width(self, value): + def origin_width(self, value: int) -> None: self.meta['grain']['cog_coded_frame']['origin_width'] = value @property - def origin_height(self): + def origin_height(self) -> int: return self.meta['grain']['cog_coded_frame']['origin_height'] @origin_height.setter - def origin_height(self, value): + def origin_height(self, value: int) -> None: self.meta['grain']['cog_coded_frame']['origin_height'] = value @property - def coded_width(self): + def coded_width(self) -> int: return self.meta['grain']['cog_coded_frame']['coded_width'] @coded_width.setter - def coded_width(self, value): + def coded_width(self, value: int) -> None: self.meta['grain']['cog_coded_frame']['coded_width'] = value @property - def coded_height(self): + def coded_height(self) -> int: return self.meta['grain']['cog_coded_frame']['coded_height'] @coded_height.setter - def coded_height(self, value): + def coded_height(self, value: int) -> None: self.meta['grain']['cog_coded_frame']['coded_height'] = value @property - def is_key_frame(self): + def is_key_frame(self) -> bool: return self.meta['grain']['cog_coded_frame']['is_key_frame'] @is_key_frame.setter - def is_key_frame(self, value): + def is_key_frame(self, value: bool) -> None: self.meta['grain']['cog_coded_frame']['is_key_frame'] = bool(value) @property - def temporal_offset(self): + def temporal_offset(self) -> int: return self.meta['grain']['cog_coded_frame']['temporal_offset'] @temporal_offset.setter - def temporal_offset(self, value): + def temporal_offset(self, value: int) -> None: self.meta['grain']['cog_coded_frame']['temporal_offset'] = value class UNITOFFSETS(MutableSequence): - def __init__(self, parent): + def __init__(self, parent: "CODEDVIDEOGRAIN"): self.parent = parent - def __getitem__(self, key): + @overload + def __getitem__(self, key: int) -> int: ... + + @overload # noqa: F811 + def __getitem__(self, key: slice) -> List[int]: ... + + def __getitem__(self, key): # noqa: F811 if 'unit_offsets' in self.parent.meta['grain']['cog_coded_frame']: return self.parent.meta['grain']['cog_coded_frame']['unit_offsets'][key] else: raise IndexError("list index out of range") - def __setitem__(self, key, value): + @overload + def __setitem__(self, key: int, value: int) -> None: ... + + @overload # noqa: F811 + def __setitem__(self, key: slice, value: Iterable[int]) -> None: ... + + def __setitem__(self, key, value): # noqa: F811 if 'unit_offsets' in self.parent.meta['grain']['cog_coded_frame']: self.parent.meta['grain']['cog_coded_frame']['unit_offsets'][key] = value else: raise IndexError("list assignment index out of range") - def __delitem__(self, key): + def __delitem__(self, key: Union[int, slice]) -> None: if 'unit_offsets' in self.parent.meta['grain']['cog_coded_frame']: - del self.parent.meta['grain']['cog_coded_frame']['unit_offsets'][key] + del cast(List[int], self.parent.meta['grain']['cog_coded_frame']['unit_offsets'])[key] if len(self.parent.meta['grain']['cog_coded_frame']['unit_offsets']) == 0: del self.parent.meta['grain']['cog_coded_frame']['unit_offsets'] else: raise IndexError("list assignment index out of range") - def insert(self, key, value): + def insert(self, key: int, value: int) -> None: if 'unit_offsets' not in self.parent.meta['grain']['cog_coded_frame']: - d = [] + d: List[int] = [] d.insert(key, value) self.parent.meta['grain']['cog_coded_frame']['unit_offsets'] = d else: - self.parent.meta['grain']['cog_coded_frame']['unit_offsets'].insert(key, value) + cast(List[int], self.parent.meta['grain']['cog_coded_frame']['unit_offsets']).insert(key, value) - def __len__(self): + def __len__(self) -> int: if 'unit_offsets' in self.parent.meta['grain']['cog_coded_frame']: return len(self.parent.meta['grain']['cog_coded_frame']['unit_offsets']) else: return 0 - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return list(self) == other - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return not (self == other) - def __repr__(self): + def __repr__(self) -> str: if 'unit_offsets' not in self.parent.meta['grain']['cog_coded_frame']: return repr([]) else: return repr(self.parent.meta['grain']['cog_coded_frame']['unit_offsets']) @property - def unit_offsets(self): + def unit_offsets(self) -> "CODEDVIDEOGRAIN.UNITOFFSETS": return CODEDVIDEOGRAIN.UNITOFFSETS(self) @unit_offsets.setter - def unit_offsets(self, value): - if value is not None and len(value) != 0: - self.meta['grain']['cog_coded_frame']['unit_offsets'] = value + def unit_offsets(self, value: Iterable[int]) -> None: + if value is not None and not (hasattr(value, "__len__") and len(cast(Sized, value)) == 0): + self.meta['grain']['cog_coded_frame']['unit_offsets'] = list(value) elif 'unit_offsets' in self.meta['grain']['cog_coded_frame']: del self.meta['grain']['cog_coded_frame']['unit_offsets'] -def size_for_audio_format(cog_audio_format, channels, samples): +def size_for_audio_format(cog_audio_format: CogAudioFormat, channels: int, samples: int) -> int: if (cog_audio_format & 0x200) == 0x200: # compressed format, no idea of correct size return 0 @@ -1303,59 +1472,64 @@ class AUDIOGRAIN(GRAIN): An integer indicating the number of samples per channel per second in this audio flow. """ - def __init__(self, meta, data): + def __init__(self, meta: AudioGrainMetadataDict, data: GrainDataParameterType): super(AUDIOGRAIN, self).__init__(meta, data) + self.meta: AudioGrainMetadataDict + self._factory = "AudioGrain" self.meta['grain']['grain_type'] = 'audio' if 'cog_audio' not in self.meta['grain']: - self.meta['grain']['cog_audio'] = {} + self.meta['grain']['cog_audio'] = {} # type: ignore if 'format' not in self.meta['grain']['cog_audio']: self.meta['grain']['cog_audio']['format'] = int(CogAudioFormat.INVALID) - for key in ['samples', 'channels', 'sample_rate']: - if key not in self.meta['grain']['cog_audio']: - self.meta['grain']['cog_audio'][key] = 0 + if 'samples' not in self.meta['grain']['cog_audio']: + self.meta['grain']['cog_audio']['samples'] = 0 + if 'channels' not in self.meta['grain']['cog_audio']: + self.meta['grain']['cog_audio']['channels'] = 0 + if 'sample_rate' not in self.meta['grain']['cog_audio']: + self.meta['grain']['cog_audio']['sample_rate'] = 0 self.meta['grain']['cog_audio']['format'] = int(self.meta['grain']['cog_audio']['format']) - def final_origin_timestamp(self): + def final_origin_timestamp(self) -> Timestamp: return (self.origin_timestamp + TimeOffset.from_count(self.samples - 1, self.sample_rate, 1)) - def normalise_time(self, value): + def normalise_time(self, value: Timestamp) -> Timestamp: return value.normalise(self.sample_rate, 1) @property - def format(self): + def format(self) -> CogAudioFormat: return CogAudioFormat(self.meta['grain']['cog_audio']['format']) @format.setter - def format(self, value): + def format(self, value: CogAudioFormat) -> None: self.meta['grain']['cog_audio']['format'] = int(value) @property - def samples(self): + def samples(self) -> int: return self.meta['grain']['cog_audio']['samples'] @samples.setter - def samples(self, value): + def samples(self, value: int) -> None: self.meta['grain']['cog_audio']['samples'] = int(value) @property - def channels(self): + def channels(self) -> int: return self.meta['grain']['cog_audio']['channels'] @channels.setter - def channels(self, value): + def channels(self, value: int) -> None: self.meta['grain']['cog_audio']['channels'] = int(value) @property - def sample_rate(self): + def sample_rate(self) -> int: return self.meta['grain']['cog_audio']['sample_rate'] @sample_rate.setter - def sample_rate(self, value): + def sample_rate(self, value: int) -> None: self.meta['grain']['cog_audio']['sample_rate'] = int(value) @property - def expected_length(self): + def expected_length(self) -> int: return size_for_audio_format(self.format, self.channels, self.samples) @@ -1436,75 +1610,80 @@ class CODEDAUDIOGRAIN(GRAIN): remainder An integer """ - def __init__(self, meta, data): + def __init__(self, meta: CodedAudioGrainMetadataDict, data: GrainDataParameterType): super(CODEDAUDIOGRAIN, self).__init__(meta, data) + self.meta: CodedAudioGrainMetadataDict + self._factory = "CodedAudioGrain" self.meta['grain']['grain_type'] = 'coded_audio' if 'cog_coded_audio' not in self.meta['grain']: - self.meta['grain']['cog_coded_audio'] = {} + self.meta['grain']['cog_coded_audio'] = {} # type: ignore if 'format' not in self.meta['grain']['cog_coded_audio']: self.meta['grain']['cog_coded_audio']['format'] = int(CogAudioFormat.INVALID) - for (key, DEF) in [('channels', 0), - ('samples', 0), - ('priming', 0), - ('remainder', 0), - ('sample_rate', 48000)]: - if key not in self.meta['grain']['cog_coded_audio']: - self.meta['grain']['cog_coded_audio'][key] = DEF + if 'channels' not in self.meta['grain']['cog_coded_audio']: + self.meta['grain']['cog_coded_audio']['channels'] = 0 + if 'samples' not in self.meta['grain']['cog_coded_audio']: + self.meta['grain']['cog_coded_audio']['samples'] = 0 + if 'priming' not in self.meta['grain']['cog_coded_audio']: + self.meta['grain']['cog_coded_audio']['priming'] = 0 + if 'remainder' not in self.meta['grain']['cog_coded_audio']: + self.meta['grain']['cog_coded_audio']['remainder'] = 0 + if 'sample_rate' not in self.meta['grain']['cog_coded_audio']: + self.meta['grain']['cog_coded_audio']['sample_rate'] = 48000 self.meta['grain']['cog_coded_audio']['format'] = int(self.meta['grain']['cog_coded_audio']['format']) - def final_origin_timestamp(self): + def final_origin_timestamp(self) -> Timestamp: return (self.origin_timestamp + TimeOffset.from_count(self.samples - 1, self.sample_rate, 1)) - def normalise_time(self, value): + def normalise_time(self, value: Timestamp) -> Timestamp: return value.normalise(self.sample_rate, 1) @property - def format(self): + def format(self) -> CogAudioFormat: return CogAudioFormat(self.meta['grain']['cog_coded_audio']['format']) @format.setter - def format(self, value): + def format(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['format'] = int(value) @property - def channels(self): + def channels(self) -> int: return self.meta['grain']['cog_coded_audio']['channels'] @channels.setter - def channels(self, value): + def channels(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['channels'] = value @property - def samples(self): + def samples(self) -> int: return self.meta['grain']['cog_coded_audio']['samples'] @samples.setter - def samples(self, value): + def samples(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['samples'] = value @property - def priming(self): + def priming(self) -> int: return self.meta['grain']['cog_coded_audio']['priming'] @priming.setter - def priming(self, value): + def priming(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['priming'] = value @property - def remainder(self): + def remainder(self) -> int: return self.meta['grain']['cog_coded_audio']['remainder'] @remainder.setter - def remainder(self, value): + def remainder(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['remainder'] = value @property - def sample_rate(self): + def sample_rate(self) -> int: return self.meta['grain']['cog_coded_audio']['sample_rate'] @sample_rate.setter - def sample_rate(self, value): + def sample_rate(self, value: int) -> None: self.meta['grain']['cog_coded_audio']['sample_rate'] = value diff --git a/mediagrains/grain_constructors.py b/mediagrains/grain_constructors.py index 1a36c16..0e289d4 100644 --- a/mediagrains/grain_constructors.py +++ b/mediagrains/grain_constructors.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -20,24 +19,61 @@ grains. """ -from __future__ import print_function -from __future__ import absolute_import - -from six import string_types - from uuid import UUID from mediatimestamp.immutable import Timestamp from fractions import Fraction +from typing import Optional, cast, Sized, List, overload +from .typing import ( + GrainDataParameterType, + GrainMetadataDict, + EmptyGrainMetadataDict, + AudioGrainMetadataDict, + CodedAudioGrainMetadataDict, + VideoGrainMetadataDict, + EventGrainMetadataDict, + CodedVideoGrainMetadataDict, + VideoGrainComponentDict) + from .cogenums import CogFrameFormat, CogFrameLayout, CogAudioFormat from .grain import GRAIN, VIDEOGRAIN, AUDIOGRAIN, CODEDVIDEOGRAIN, CODEDAUDIOGRAIN, EVENTGRAIN, size_for_audio_format + __all__ = ["Grain", "VideoGrain", "AudioGrain", "CodedVideoGrain", "CodedAudioGrain", "EventGrain"] -def Grain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, - sync_timestamp=None, creation_timestamp=None, rate=Fraction(0, 1), duration=Fraction(0, 1), - flow_id=None, data=None, src_id=None, source_id=None, meta=None): +@overload +def Grain(src_id_or_meta: GrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> GRAIN: ... + + +@overload +def Grain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(0, 1), + duration: Fraction = Fraction(0, 1), + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + meta: Optional[GrainMetadataDict] = None) -> GRAIN: ... + + +def Grain(src_id_or_meta=None, + flow_id_or_data=None, + origin_timestamp=None, + sync_timestamp=None, + creation_timestamp=None, + rate=Fraction(0, 1), + duration=Fraction(0, 1), + flow_id=None, + data=None, + src_id=None, + source_id=None, + meta=None): """\ Function called to construct a grain either from existing data or with new data. @@ -53,6 +89,10 @@ def Grain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, function, otherwise a generic grain object will be returned which wraps the meta and data elements. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for a Grain should look like: { @@ -99,12 +139,12 @@ class mediagrains.grain.GRAIN if meta is None: if isinstance(src_id_or_meta, dict): meta = src_id_or_meta - if data is None: + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if meta is None: @@ -123,51 +163,75 @@ class mediagrains.grain.GRAIN if src_id is None or flow_id is None: raise AttributeError("Must specify at least meta or src_id and flow_id") - if isinstance(src_id, UUID): - src_id = str(src_id) - if isinstance(flow_id, UUID): - flow_id = str(flow_id) - - if not isinstance(src_id, string_types) or not isinstance(flow_id, string_types): + if not isinstance(src_id, UUID) or not isinstance(flow_id, UUID): raise AttributeError("Invalid types for src_id and flow_id") - meta = { + meta = EmptyGrainMetadataDict({ "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { - "grain_type": "empty", - "source_id": src_id, - "flow_id": flow_id, - "origin_timestamp": str(ots), - "sync_timestamp": str(sts), - "creation_timestamp": str(cts), - "rate": { - "numerator": Fraction(rate).numerator, - "denominator": Fraction(rate).denominator, - }, - "duration": { - "numerator": Fraction(duration).numerator, - "denominator": Fraction(duration).denominator, - }, + 'grain_type': "empty", + 'source_id': str(src_id), + 'flow_id': str(flow_id), + 'origin_timestamp': str(ots), + 'sync_timestamp': str(sts), + 'creation_timestamp': str(cts), + 'rate': { + 'numerator': Fraction(rate).numerator, + 'denominator': Fraction(rate).denominator + }, + 'duration': { + 'numerator': Fraction(duration).numerator, + 'denominator': Fraction(duration).denominator } } + }) data = None if 'grain' in meta and 'grain_type' in meta['grain'] and meta['grain']['grain_type'] == 'video': - return VideoGrain(meta, data) + return VideoGrain(cast(VideoGrainMetadataDict, meta), data) elif 'grain' in meta and 'grain_type' in meta['grain'] and meta['grain']['grain_type'] == 'audio': - return AudioGrain(meta, data) + return AudioGrain(cast(AudioGrainMetadataDict, meta), data) elif 'grain' in meta and 'grain_type' in meta['grain'] and meta['grain']['grain_type'] == 'coded_video': - return CodedVideoGrain(meta, data) + return CodedVideoGrain(cast(CodedVideoGrainMetadataDict, meta), data) elif 'grain' in meta and 'grain_type' in meta['grain'] and meta['grain']['grain_type'] == 'coded_audio': - return CodedAudioGrain(meta, data) + return CodedAudioGrain(cast(CodedAudioGrainMetadataDict, meta), data) elif 'grain' in meta and 'grain_type' in meta['grain'] and meta['grain']['grain_type'] in ['event', 'data']: - return EventGrain(meta, data) + return EventGrain(cast(EventGrainMetadataDict, meta), data) else: return GRAIN(meta, data) -def AudioGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, - sync_timestamp=None, creation_timestamp=None, rate=Fraction(25, 1), duration=Fraction(1, 25), +@overload +def AudioGrain(src_id_or_meta: AudioGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> AUDIOGRAIN: ... + + +@overload +def AudioGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + cog_audio_format: CogAudioFormat = CogAudioFormat.INVALID, + samples: int = 0, + channels: int = 0, + sample_rate: int = 48000, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + format: Optional[CogAudioFormat] = None, + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None) -> AUDIOGRAIN: ... + + +def AudioGrain(src_id_or_meta=None, + flow_id_or_data=None, + origin_timestamp=None, + sync_timestamp=None, + creation_timestamp=None, + rate=Fraction(25, 1), + duration=Fraction(1, 25), cog_audio_format=CogAudioFormat.INVALID, samples=0, channels=0, @@ -187,6 +251,10 @@ def AudioGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, where meta is a dictionary containing the grain metadata, and data is a bytes-like object which contains the grain's payload. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for an Audio Grain should look like: { @@ -240,7 +308,7 @@ class mediagrains.grain.AUDIOGRAIN (the parameters "source_id" and "src_id" are aliases for each other. source_id is probably prefered, but src_id is kept avaialble for backwards compatibility) """ - meta = None + meta: Optional[AudioGrainMetadataDict] = None if cog_audio_format is None: cog_audio_format = format @@ -248,13 +316,13 @@ class mediagrains.grain.AUDIOGRAIN src_id = source_id if isinstance(src_id_or_meta, dict): - meta = src_id_or_meta - if data is None: + meta = cast(AudioGrainMetadataDict, src_id_or_meta) + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if meta is None: @@ -271,21 +339,21 @@ class mediagrains.grain.AUDIOGRAIN meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { - "grain_type": "audio", - "source_id": str(src_id), - "flow_id": str(flow_id), - "origin_timestamp": str(origin_timestamp), - "sync_timestamp": str(sync_timestamp), - "creation_timestamp": str(cts), - "rate": { - "numerator": Fraction(rate).numerator, - "denominator": Fraction(rate).denominator, - }, - "duration": { - "numerator": Fraction(duration).numerator, - "denominator": Fraction(duration).denominator, - }, - "cog_audio": { + 'grain_type': "audio", + 'source_id': str(src_id), + 'flow_id': str(flow_id), + 'origin_timestamp': str(origin_timestamp), + 'sync_timestamp': str(sync_timestamp), + 'creation_timestamp': str(cts), + 'rate': { + 'numerator': Fraction(rate).numerator, + 'denominator': Fraction(rate).denominator + }, + 'duration': { + 'numerator': Fraction(duration).numerator, + 'denominator': Fraction(duration).denominator + }, + 'cog_audio': { "format": cog_audio_format, "samples": samples, "channels": channels, @@ -301,6 +369,33 @@ class mediagrains.grain.AUDIOGRAIN return AUDIOGRAIN(meta, data) +@overload +def CodedAudioGrain(src_id_or_meta: CodedAudioGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> CODEDAUDIOGRAIN: ... + + +@overload +def CodedAudioGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + cog_audio_format: CogAudioFormat = CogAudioFormat.INVALID, + samples: int = 0, + channels: int = 0, + priming: int = 0, + remainder: int = 0, + sample_rate: int = 48000, + length: Optional[int] = None, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + format: Optional[CogAudioFormat] = None, + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None) -> CODEDAUDIOGRAIN: ... + + def CodedAudioGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, @@ -318,7 +413,8 @@ def CodedAudioGrain(src_id_or_meta=None, src_id=None, source_id=None, format=None, - flow_id=None, data=None): + flow_id=None, + data=None): """\ Function called to construct a coded audio grain either from existing data or with new data. @@ -329,12 +425,16 @@ def CodedAudioGrain(src_id_or_meta=None, where meta is a dictionary containing the grain metadata, and data is a bytes-like object which contains the grain's payload. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for a Coded Audio Grain should look like: { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { - "grain_type": "audio", + "grain_type": "coded_audio", "source_id": src_id, # str or uuid.UUID "flow_id": flow_id, # str or uuid.UUID "origin_timestamp": origin_timestamp, # str or mediatimestamps.Timestamp @@ -387,7 +487,7 @@ class mediagrains.grain.CODEDAUDIOGRAIN but src_id is kept avaialble for backwards compatibility) """ - meta = None + meta: Optional[CodedAudioGrainMetadataDict] = None if source_id is not None: src_id = source_id @@ -396,18 +496,18 @@ class mediagrains.grain.CODEDAUDIOGRAIN cog_audio_format = format if isinstance(src_id_or_meta, dict): - meta = src_id_or_meta - if data is None: + meta = cast(CodedAudioGrainMetadataDict, src_id_or_meta) + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if length is None: - if data is not None: - length = len(data) + if data is not None and hasattr(data, "__len__"): + length = len(cast(Sized, data)) else: length = 0 @@ -423,21 +523,21 @@ class mediagrains.grain.CODEDAUDIOGRAIN meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { - "grain_type": "coded_audio", - "source_id": str(src_id), - "flow_id": str(flow_id), - "origin_timestamp": str(origin_timestamp), - "sync_timestamp": str(sync_timestamp), - "creation_timestamp": str(cts), - "rate": { - "numerator": Fraction(rate).numerator, - "denominator": Fraction(rate).denominator, - }, - "duration": { - "numerator": Fraction(duration).numerator, - "denominator": Fraction(duration).denominator, - }, - "cog_coded_audio": { + 'grain_type': "coded_audio", + 'source_id': str(src_id), + 'flow_id': str(flow_id), + 'origin_timestamp': str(origin_timestamp), + 'sync_timestamp': str(sync_timestamp), + 'creation_timestamp': str(cts), + 'rate': { + 'numerator': Fraction(rate).numerator, + 'denominator': Fraction(rate).denominator + }, + 'duration': { + 'numerator': Fraction(duration).numerator, + 'denominator': Fraction(duration).denominator + }, + 'cog_coded_audio': { "format": cog_audio_format, "samples": samples, "channels": channels, @@ -454,11 +554,48 @@ class mediagrains.grain.CODEDAUDIOGRAIN return CODEDAUDIOGRAIN(meta, data) -def VideoGrain(src_id_or_meta=None, flow_id_or_data=None, creation_timestamp=None, origin_timestamp=None, - sync_timestamp=None, rate=Fraction(25, 1), duration=Fraction(1, 25), - cog_frame_format=CogFrameFormat.UNKNOWN, width=1920, - height=1080, cog_frame_layout=CogFrameLayout.UNKNOWN, - src_id=None, source_id=None, format=None, layout=None, flow_id=None, data=None): +@overload +def VideoGrain(src_id_or_meta: VideoGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> VIDEOGRAIN: ... + + +@overload +def VideoGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + cog_frame_format: CogFrameFormat = CogFrameFormat.UNKNOWN, + width: int = 1920, + height: int = 1080, + cog_frame_layout: CogFrameLayout = CogFrameLayout.UNKNOWN, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + format: Optional[CogFrameFormat] = None, + layout: Optional[CogFrameLayout] = None, + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None) -> VIDEOGRAIN: ... + + +def VideoGrain(src_id_or_meta=None, + flow_id_or_data=None, + origin_timestamp=None, + creation_timestamp=None, + sync_timestamp=None, + rate=Fraction(25, 1), + duration=Fraction(1, 25), + cog_frame_format=CogFrameFormat.UNKNOWN, + width=1920, + height=1080, + cog_frame_layout=CogFrameLayout.UNKNOWN, + src_id=None, + source_id=None, + format=None, + layout=None, + flow_id=None, + data=None): """\ Function called to construct a video grain either from existing data or with new data. @@ -469,6 +606,10 @@ def VideoGrain(src_id_or_meta=None, flow_id_or_data=None, creation_timestamp=Non where meta is a dictionary containing the grain metadata, and data is a bytes-like object which contains the grain's payload. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for a Video Grain should look like: { @@ -544,7 +685,7 @@ class mediagrains.grain.VIDEOGRAIN (the parameters "source_id" and "src_id" are aliases for each other. source_id is probably prefered, but src_id is kept avaialble for backwards compatibility) """ - meta = None + meta: Optional[VideoGrainMetadataDict] = None if cog_frame_format is None: cog_frame_format = format @@ -554,13 +695,13 @@ class mediagrains.grain.VIDEOGRAIN cog_frame_layout = layout if isinstance(src_id_or_meta, dict): - meta = src_id_or_meta - if data is None: + meta = cast(VideoGrainMetadataDict, src_id_or_meta) + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if meta is None: @@ -577,21 +718,21 @@ class mediagrains.grain.VIDEOGRAIN meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { - "grain_type": "video", - "source_id": str(src_id), - "flow_id": str(flow_id), - "origin_timestamp": str(origin_timestamp), - "sync_timestamp": str(sync_timestamp), - "creation_timestamp": str(cts), - "rate": { - "numerator": Fraction(rate).numerator, - "denominator": Fraction(rate).denominator, - }, - "duration": { - "numerator": Fraction(duration).numerator, - "denominator": Fraction(duration).denominator, - }, - "cog_frame": { + 'grain_type': "video", + 'source_id': str(src_id), + 'flow_id': str(flow_id), + 'origin_timestamp': str(origin_timestamp), + 'sync_timestamp': str(sync_timestamp), + 'creation_timestamp': str(cts), + 'rate': { + 'numerator': Fraction(rate).numerator, + 'denominator': Fraction(rate).denominator, + }, + 'duration': { + 'numerator': Fraction(duration).numerator, + 'denominator': Fraction(duration).denominator, + }, + 'cog_frame': { "format": cog_frame_format, "width": width, "height": height, @@ -602,7 +743,7 @@ class mediagrains.grain.VIDEOGRAIN }, } - def size_for_format(fmt, w, h): + def size_for_format(fmt: CogFrameFormat, w: int, h: int) -> int: if ((fmt >> 8) & 0x1) == 0x00: # Cog frame is not packed h_shift = (fmt & 0x01) v_shift = ((fmt >> 1) & 0x01) @@ -642,8 +783,8 @@ def size_for_format(fmt, w, h): size = size_for_format(cog_frame_format, width, height) data = bytearray(size) - def components_for_format(fmt, w, h): - components = [] + def components_for_format(fmt: CogFrameFormat, w: int, h: int) -> List[VideoGrainComponentDict]: + components: List[VideoGrainComponentDict] = [] if ((fmt >> 8) & 0x1) == 0x00: # Cog frame is not packed h_shift = (fmt & 0x01) v_shift = ((fmt >> 1) & 0x01) @@ -735,19 +876,68 @@ def components_for_format(fmt, w, h): }) return components - if "cog_frame" in meta['grain'] and ("components" not in meta['grain']['cog_frame'] or len(meta['grain']['cog_frame']['components']) == 0): + if ("cog_frame" in meta['grain'] and + ("components" not in meta['grain']['cog_frame'] or + len(meta['grain']['cog_frame']['components']) == 0)): meta['grain']['cog_frame']['components'] = components_for_format(cog_frame_format, width, height) return VIDEOGRAIN(meta, data) -def CodedVideoGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, creation_timestamp=None, - sync_timestamp=None, rate=Fraction(25, 1), duration=Fraction(1, 25), - cog_frame_format=CogFrameFormat.UNKNOWN, origin_width=1920, - origin_height=1080, coded_width=None, - coded_height=None, is_key_frame=False, temporal_offset=0, length=None, - cog_frame_layout=CogFrameLayout.UNKNOWN, unit_offsets=None, - flow_id=None, src_id=None, source_id=None, format=None, layout=None, data=None): +@overload +def CodedVideoGrain(src_id_or_meta: CodedVideoGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> CODEDVIDEOGRAIN: ... + + +@overload +def CodedVideoGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + cog_frame_format: CogFrameFormat = CogFrameFormat.UNKNOWN, + origin_width: int = 1920, + origin_height: int = 1080, + coded_width: Optional[int] = None, + coded_height: Optional[int] = None, + is_key_frame: bool = False, + temporal_offset: int = 0, + length: Optional[int] = None, + cog_frame_layout: CogFrameLayout = CogFrameLayout.UNKNOWN, + unit_offsets: Optional[List[int]] = None, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + format: Optional[CogFrameFormat] = None, + layout: Optional[CogFrameLayout] = None, + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None) -> CODEDVIDEOGRAIN: ... + + +def CodedVideoGrain(src_id_or_meta=None, + flow_id_or_data=None, + origin_timestamp=None, + creation_timestamp=None, + sync_timestamp=None, + rate=Fraction(25, 1), + duration=Fraction(1, 25), + cog_frame_format=CogFrameFormat.UNKNOWN, + origin_width=1920, + origin_height=1080, + coded_width=None, + coded_height=None, + is_key_frame=False, + temporal_offset=0, + length=None, + cog_frame_layout=CogFrameLayout.UNKNOWN, + unit_offsets=None, + src_id=None, + source_id=None, + format=None, + layout=None, + flow_id=None, + data=None): """\ Function called to construct a coded video grain either from existing data or with new data. @@ -758,6 +948,10 @@ def CodedVideoGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp= where meta is a dictionary containing the grain metadata, and data is a bytes-like object which contains the grain's payload. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for a Video Grain should look like: { @@ -821,7 +1015,7 @@ class mediagrains.grain.CODEDVIDEOGRAIN (the parameters "source_id" and "src_id" are aliases for each other. source_id is probably prefered, but src_id is kept avaialble for backwards compatibility) """ - meta = None + meta: Optional[CodedVideoGrainMetadataDict] = None if cog_frame_format is None: cog_frame_format = format @@ -831,13 +1025,13 @@ class mediagrains.grain.CODEDVIDEOGRAIN cog_frame_layout = layout if isinstance(src_id_or_meta, dict): - meta = src_id_or_meta - if data is None: + meta = cast(CodedVideoGrainMetadataDict, src_id_or_meta) + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if coded_width is None: @@ -846,8 +1040,8 @@ class mediagrains.grain.CODEDVIDEOGRAIN coded_height = origin_height if length is None: - if data is not None: - length = len(data) + if data is not None and hasattr(data, "__len__"): + length = len(cast(Sized, data)) else: length = 0 @@ -902,10 +1096,42 @@ class mediagrains.grain.CODEDVIDEOGRAIN return CODEDVIDEOGRAIN(meta, data) -def EventGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, - sync_timestamp=None, creation_timestamp=None, rate=Fraction(25, 1), duration=Fraction(1, 25), - event_type='', topic='', - src_id=None, source_id=None, flow_id=None, meta=None, data=None): +@overload +def EventGrain(src_id_or_meta: EventGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> EVENTGRAIN: ... + + +@overload +def EventGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + event_type: str = '', + topic: str = '', + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + flow_id: Optional[UUID] = None, + meta: Optional[EventGrainMetadataDict] = None, + data: GrainDataParameterType = None) -> EVENTGRAIN: ... + + +def EventGrain(src_id_or_meta=None, + flow_id_or_data=None, + origin_timestamp=None, + creation_timestamp=None, + sync_timestamp=None, + rate=Fraction(25, 1), + duration=Fraction(1, 25), + event_type='', + topic='', + src_id=None, + source_id=None, + flow_id=None, + meta=None, + data=None): """\ Function called to construct an event grain either from existing data or with new data. @@ -916,6 +1142,10 @@ def EventGrain(src_id_or_meta=None, flow_id_or_data=None, origin_timestamp=None, where meta is a dictionary containing the grain metadata, and data is a bytes-like object which contains a string representation of the json grain payload. +Optionally the data element can be replaced with an Awaitable that will return a +data element when awaited. This is useful for grains that are backed with some +sort of asynchronous IO system. + A properly formated metadata dictionary for an Event Grain should look like: { @@ -980,14 +1210,14 @@ class mediagrains.grain.EVENTGRAIN src_id = source_id if isinstance(src_id_or_meta, dict): - if meta is None: + if meta is None and not isinstance(src_id_or_meta, UUID): meta = src_id_or_meta - if data is None: + if data is None and not isinstance(flow_id_or_data, UUID): data = flow_id_or_data else: - if src_id is None: + if src_id is None and isinstance(src_id_or_meta, UUID): src_id = src_id_or_meta - if flow_id is None: + if flow_id is None and isinstance(flow_id_or_data, UUID): flow_id = flow_id_or_data if meta is None: @@ -1001,7 +1231,7 @@ class mediagrains.grain.EVENTGRAIN origin_timestamp = cts if sync_timestamp is None: sync_timestamp = origin_timestamp - meta = { + meta = EventGrainMetadataDict({ "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { "grain_type": "event", @@ -1024,6 +1254,6 @@ class mediagrains.grain.EVENTGRAIN "data": [] } }, - } + }) return EVENTGRAIN(meta, data) diff --git a/mediagrains/gsf.py b/mediagrains/gsf.py index 4bf668a..cd0ece0 100644 --- a/mediagrains/gsf.py +++ b/mediagrains/gsf.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -20,17 +19,49 @@ objects. """ -from __future__ import print_function, absolute_import from . import Grain -from six import indexbytes from uuid import UUID, uuid1 from datetime import datetime +from io import BytesIO, RawIOBase, BufferedIOBase from mediatimestamp.immutable import Timestamp from fractions import Fraction from frozendict import frozendict -from six import BytesIO, PY3 from .utils import IOBytes -from os import SEEK_SET +from .utils.synchronise import run_awaitable_synchronously, Synchronised +from os import SEEK_SET, SEEK_CUR +import warnings + +from inspect import isawaitable + +from typing import ( + Callable, + Optional, + Iterable, + Tuple, + List, + Dict, + Mapping, + cast, + Union, + Type, + IO, + Sequence, + AsyncIterable, + Awaitable, + overload) +from typing_extensions import TypedDict +from .typing import GrainMetadataDict, GrainDataParameterType, RationalTypes + +from .grain import GRAIN, VIDEOGRAIN, EVENTGRAIN, AUDIOGRAIN, CODEDAUDIOGRAIN, CODEDVIDEOGRAIN + +from .utils.asyncbinaryio import AsyncBinaryIO, OpenAsyncBinaryIO, AsyncFileWrapper, OpenAsyncFileWrapper + +from contextlib import contextmanager + +from deprecated import deprecated + +from enum import Enum + __all__ = ["GSFDecoder", "load", "loads", "GSFError", "GSFDecodeError", "GSFDecodeBadFileTypeError", "GSFDecodeBadVersionError", @@ -38,7 +69,23 @@ "GSFEncodeAddToActiveDump"] -def loads(s, cls=None, parse_grain=None, **kwargs): +@contextmanager +def no_deprecation_warnings(): + with warnings.catch_warnings(record=True) as warns: + yield + + for w in warns: + if w.category != DeprecationWarning: + warnings.showwarning(w.message, w.category, w.filename, w.lineno) + + +GSFFileHeaderDict = dict + + +def loads(s: bytes, + cls: Optional[Type["GSFDecoder"]] = None, + parse_grain: Optional[Callable[[GrainMetadataDict, GrainDataParameterType], GRAIN]] = None, + **kwargs) -> Tuple[GSFFileHeaderDict, Dict[int, List[GRAIN]]]: """Deserialise a GSF file from a string (or similar) into python, returns a pair of (head, segments) where head is a python dict containing general metadata from the file, and segments is a dictionary @@ -52,25 +99,56 @@ def loads(s, cls=None, parse_grain=None, **kwargs): cls = GSFDecoder if parse_grain is None: parse_grain = Grain - dec = cls(parse_grain=parse_grain, **kwargs) - return dec.decode(s) + return load(BytesIO(s), cls=cls, parse_grain=parse_grain) + + +@overload +def load(fp: IO[bytes], + cls: Optional[Type["GSFDecoder"]] = None, + parse_grain: Optional[Callable[[GrainMetadataDict, GrainDataParameterType], GRAIN]] = None, + **kwargs) -> Tuple[GSFFileHeaderDict, Dict[int, List[GRAIN]]]: ... -def load(fp, cls=None, parse_grain=None, **kwargs): + +@overload +def load(fp: AsyncBinaryIO, + cls: Optional[Type["GSFDecoder"]] = None, + parse_grain: Optional[Callable[[GrainMetadataDict, GrainDataParameterType], GRAIN]] = None, + **kwargs) -> Awaitable[Tuple[GSFFileHeaderDict, Dict[int, List[GRAIN]]]]: ... + + +def load(fp, + cls=None, + parse_grain=None, + **kwargs): """Deserialise a GSF file from a file object (or similar) into python, returns a pair of (head, segments) where head is a python dict containing general metadata from the file, and segments is a dictionary mapping numeric segment ids to lists of Grain objects. + If an asynchronous file object is provided then return an awaitable that + will do the above. + If you wish to use a custom GSFDecoder subclass pass it as cls, if you wish to use a custom Grain constructor pass it as parse_grain. The defaults are GSFDecoder and Grain. Extra kwargs will be passed to the decoder constructor.""" - s = fp.read() - return loads(s, cls=cls, parse_grain=parse_grain, **kwargs) + if cls is None: + cls = GSFDecoder + if parse_grain is None: + parse_grain = Grain + + if isinstance(fp, AsyncBinaryIO): + return cls(file_data=fp, parse_grain=parse_grain, **kwargs)._asynchronously_decode() + else: + return run_awaitable_synchronously(cls(file_data=AsyncFileWrapper(fp), parse_grain=parse_grain, **kwargs)._asynchronously_decode()) -def dump(grains, fp, cls=None, segment_tags=None, **kwargs): +def dump(grains: Iterable[GRAIN], + fp: IO[bytes], + cls: Optional[Type["GSFEncoder"]] = None, + segment_tags: Optional[Iterable[Tuple[str, str]]] = None, + **kwargs) -> None: """Serialise a series of grains into a GSF file. :param grains an iterable of grain objects @@ -83,13 +161,16 @@ def dump(grains, fp, cls=None, segment_tags=None, **kwargs): This method will serialise the grains in a single segment.""" if cls is None: cls = GSFEncoder - enc = cls(fp, **kwargs) - seg = enc.add_segment(tags=segment_tags) - seg.add_grains(grains) - enc.dump() + with cls(fp, **kwargs) as enc: + seg = enc.add_segment(tags=segment_tags) + seg.add_grains(grains) -def dumps(grains, cls=None, segment_tags=None, **kwargs): + +def dumps(grains: Iterable[GRAIN], + cls: Optional[Type["GSFEncoder"]] = None, + segment_tags: Optional[Iterable[Tuple[str, str]]] = None, + **kwargs) -> bytes: """Serialise a series of grains into a new bytes object. :param grains an iterable of grain objects @@ -153,14 +234,14 @@ def __init__(self, msg, i, major, minor): self.minor = minor -class GSFBlock(): +class AsyncGSFBlock(): """A single block in a GSF file Has methods to read various types from the block. - Can also be used as a context manager, in which case it will automatically decode the block tag and size, exposed + Can also be used as an async context manager, in which case it will automatically decode the block tag and size, exposed by the `tag` and `size` attributes. """ - def __init__(self, file_data, want_tag=None, raise_on_wrong_tag=False): + def __init__(self, file_data: OpenAsyncBinaryIO, want_tag: Optional[str] = None, raise_on_wrong_tag: bool = False): """Constructor. Records the start byte of the block in `block_start` :param file_data: An instance of io.BufferedReader positioned at the start of the block @@ -171,10 +252,10 @@ def __init__(self, file_data, want_tag=None, raise_on_wrong_tag=False): self.want_tag = want_tag self.raise_on_wrong_tag = raise_on_wrong_tag - self.size = None + self.size: Optional[int] = None self.block_start = self.file_data.tell() # In binary mode, this should always be in bytes - def __enter__(self): + async def __aenter__(self) -> "AsyncGSFBlock": """When used as a context manager, read block size and tag on entry - When entering a block, tag and size should be read @@ -186,7 +267,7 @@ def __enter__(self): :raises GSFDecodeError: If the block tag failed to decode as UTF-8, or an unwanted tag was found """ while True: - tag_bytes = self.file_data.read(4) + tag_bytes = await self.file_data.read(4) try: self.tag = tag_bytes.decode(encoding="utf-8") @@ -196,7 +277,7 @@ def __enter__(self): self.block_start ) - self.size = self.read_uint(4) + self.size = await self.read_uint(4) if self.want_tag is None or self.tag == self.want_tag: return self @@ -207,9 +288,12 @@ def __enter__(self): self.file_data.seek(self.block_start + self.size, SEEK_SET) self.block_start = self.file_data.tell() - def __exit__(self, *args): + async def __aexit__(self, *args): """When used as a context manager, exiting context should seek to the block end""" - self.file_data.seek(self.block_start + self.size, SEEK_SET) + try: + self.file_data.seek(self.block_start + self.size, SEEK_SET) + except Exception: + pass def has_child_block(self, strict_blocks=True): """Checks if there is space for another child block in this block @@ -235,7 +319,7 @@ def has_child_block(self, strict_blocks=True): else: return False - def child_blocks(self, strict_blocks=True): + async def child_blocks(self, strict_blocks=True): """Generator for each child block - each yielded block sits within the context manager Must be used in a context manager. @@ -245,7 +329,7 @@ def child_blocks(self, strict_blocks=True): :raises GSFDecodeError: If there is a partial block and strict=True """ while self.has_child_block(strict_blocks=strict_blocks): - with GSFBlock(self.file_data) as child_block: + async with AsyncGSFBlock(self.file_data) as child_block: yield child_block def get_remaining(self): @@ -258,7 +342,7 @@ def get_remaining(self): assert self.size is not None, "get_remaining() only works in a context manager" return (self.block_start + self.size) - self.file_data.tell() - def read_uint(self, length): + async def read_uint(self, length) -> int: """Read an unsigned integer of length `length` :param length: Number of bytes used to store the integer @@ -266,49 +350,49 @@ def read_uint(self, length): :raises EOFError: If there are fewer than `length` bytes left in the source """ r = 0 - uint_bytes = bytes(self.file_data.read(length)) + uint_bytes = await self.file_data.read(length) if len(uint_bytes) != length: raise EOFError("Unable to read enough bytes from source") for n in range(0, length): - r += (indexbytes(uint_bytes, n) << (n*8)) + r += (uint_bytes[n] << (n*8)) return r - def read_bool(self): + async def read_bool(self): """Read a boolean value :returns: Boolean value :raises EOFError: If there are no more bytes left in the source""" - n = self.read_uint(1) + n = await self.read_uint(1) return (n != 0) - def read_sint(self, length): + async def read_sint(self, length: int) -> int: """Read a 2's complement signed integer :param length: Number of bytes used to store the integer :returns: Signed integer :raises EOFError: If there are fewer than `length` bytes left in the source """ - r = self.read_uint(length) + r = await self.read_uint(length) if (r >> ((8*length) - 1)) == 1: r -= (1 << (8*length)) return r - def read_string(self, length): + async def read_string(self, length: int) -> str: """Read a fixed-length string, treating it as UTF-8 :param length: Number of bytes in the string :returns: String :raises EOFError: If there are fewer than `length` bytes left in the source """ - string_data = self.file_data.read(length) + string_data = await self.file_data.read(length) if (len(string_data) != length): raise EOFError("Unable to read enough bytes from source") return string_data.decode(encoding='utf-8') - def read_varstring(self): + async def read_varstring(self) -> str: """Read a variable length string Reads a 2 byte uint to get the string length, then reads a string of that length @@ -316,47 +400,47 @@ def read_varstring(self): :returns: String :raises EOFError: If there are too few bytes left in the source """ - length = self.read_uint(2) - return self.read_string(length) + length = await self.read_uint(2) + return await self.read_string(length) - def read_uuid(self): + async def read_uuid(self) -> UUID: """Read a UUID :returns: UUID :raises EOFError: If there are fewer than l bytes left in the source """ - uuid_data = self.file_data.read(16) + uuid_data = await self.file_data.read(16) if (len(uuid_data) != 16): raise EOFError("Unable to read enough bytes from source") return UUID(bytes=uuid_data) - def read_timestamp(self): + async def read_timestamp(self) -> datetime: """Read a date-time (with seconds resolution) stored in 7 bytes :returns: Datetime :raises EOFError: If there are fewer than 7 bytes left in the source """ - year = self.read_sint(2) - month = self.read_uint(1) - day = self.read_uint(1) - hour = self.read_uint(1) - minute = self.read_uint(1) - second = self.read_uint(1) + year = await self.read_sint(2) + month = await self.read_uint(1) + day = await self.read_uint(1) + hour = await self.read_uint(1) + minute = await self.read_uint(1) + second = await self.read_uint(1) return datetime(year, month, day, hour, minute, second) - def read_ippts(self): + async def read_ippts(self) -> Timestamp: """Read a mediatimestamp.Timestamp :returns: Timestamp :raises EOFError: If there are fewer than 10 bytes left in the source """ - secs = self.read_uint(6) - nano = self.read_uint(4) + secs = await self.read_uint(6) + nano = await self.read_uint(4) return Timestamp(secs, nano) - def read_rational(self): + async def read_rational(self) -> Fraction: """Read a rational (fraction) If numerator or denominator is 0, returns Fraction(0) @@ -364,104 +448,132 @@ def read_rational(self): :returns: fraction.Fraction :raises EOFError: If there are fewer than 8 bytes left in the source """ - numerator = self.read_uint(4) - denominator = self.read_uint(4) + numerator = await self.read_uint(4) + denominator = await self.read_uint(4) if numerator == 0 or denominator == 0: return Fraction(0) else: return Fraction(numerator, denominator) -class GSFDecoder(object): - """A decoder for GSF format. +class GrainDataLoadingMode (Enum): + """This enumeration describes the mode for loading grains from the input. + + For a Non-seekable input: + LOAD_IMMEDIATELY -- Grain data will be read as the stream is processed + ALWAYS_LOAD_DEFER_IF_POSSIBLE -- Grain data will be read as the stream is processed + ALWAYS_DEFER_LOAD_IF_POSSIBLE -- Grain data will be read as the stream is processed + LOAD_NEVER -- Grain data will be skipped over + + For a Seekable input: + LOAD_IMMEDIATELY -- Grain data will be read as the stream is processed + ALWAYS_LOAD_DEFER_IF_POSSIBLE -- Grain data will be skipped initially, but loaded + upon request. All unloaded grains will be loaded when + the context manager is exited. + ALWAYS_DEFER_LOAD_IF_POSSIBLE -- Grain data will be skipped initially, but loaded + upon request. All unloaded grains will have their data + loading canceled when the context manager is exited. + LOAD_NEVER -- Grain data will be skipped over + """ + LOAD_IMMEDIATELY = 0 + ALWAYS_LOAD_DEFER_IF_POSSIBLE = 1 + ALWAYS_DEFER_LOAD_IF_POSSIBLE = 2 + LOAD_NEVER = 3 - Provides methods to decode the header of a GSF file, followed by a generator to get each grain, wrapped in some - grain method (mediagrains.Grain by default.) - Can also be used to make a one-off decode of a GSF file from a bytes-like object by calling `decode(bytes_like)`. - """ - def __init__(self, parse_grain=Grain, file_data=None, **kwargs): - """Constructor +class GSFAsyncDecoderSession(object): + def __init__(self, + parse_grain: Callable[[GrainMetadataDict, GrainDataParameterType], GRAIN], + file_data: OpenAsyncBinaryIO, + sync_compatibility_mode: bool): + self.file_data = file_data + + if not self.file_data.seekable_forwards(): + raise RuntimeError("Cannot decode a stream that is not at least forward seekable") - :param parse_grain: Function that takes a (metadata dict, buffer) and returns a grain representation - :param file_data: BufferedReader (or similar) containing GSF data to decode - """ self.Grain = parse_grain - self.file_data = file_data + self.file_headers: Optional[GSFFileHeaderDict] = None + + self._exiting = False + self._unloaded_lazy_grains: Dict[int, GRAIN] = {} + self._next_lazy_grain_number = 0 - def _decode_ssb_header(self): + self._sync_compatibility_mode = sync_compatibility_mode + + async def _decode_ssb_header(self): """Find and read the SSB header in the GSF file :returns: (major, minor) version tuple :raises GSFDecodeBadFileTypeError: If the SSB tag shows this isn't a GSF file """ - ssb_block = GSFBlock(self.file_data) + ssb_block = AsyncGSFBlock(self.file_data) - tag = ssb_block.read_string(8) + tag = await ssb_block.read_string(8) if tag != "SSBBgrsg": raise GSFDecodeBadFileTypeError("File lacks correct header", ssb_block.block_start, tag) - major = ssb_block.read_uint(2) - minor = ssb_block.read_uint(2) + major = await ssb_block.read_uint(2) + minor = await ssb_block.read_uint(2) return (major, minor) - def _decode_head(self, head_block): + async def _decode_head(self, + head_block: AsyncGSFBlock) -> GSFFileHeaderDict: """Decode the "head" block and extract ID, created date, segments and tags :param head_block: GSFBlock representing the "head" block :returns: Head block as a dict """ - head = {} - head['id'] = head_block.read_uuid() - head['created'] = head_block.read_timestamp() + head: GSFFileHeaderDict = {} + head['id'] = await head_block.read_uuid() + head['created'] = await head_block.read_timestamp() head['segments'] = [] head['tags'] = [] # Read head block children - for head_child in head_block.child_blocks(): + async for head_child in head_block.child_blocks(): # Parse a segment block if head_child.tag == "segm": segm = {} - segm['local_id'] = head_child.read_uint(2) - segm['id'] = head_child.read_uuid() - segm['count'] = head_child.read_sint(8) + segm['local_id'] = await head_child.read_uint(2) + segm['id'] = await head_child.read_uuid() + segm['count'] = await head_child.read_sint(8) segm['tags'] = [] # Segment blocks can have child tags as well while head_child.has_child_block(): - with GSFBlock(self.file_data) as segm_tag: + async with AsyncGSFBlock(self.file_data) as segm_tag: if segm_tag.tag == "tag ": - key = segm_tag.read_varstring() - value = segm_tag.read_varstring() + key = await segm_tag.read_varstring() + value = await segm_tag.read_varstring() segm['tags'].append((key, value)) head['segments'].append(segm) # Parse a tag block elif head_child.tag == "tag ": - key = head_child.read_varstring() - value = head_child.read_varstring() + key = await head_child.read_varstring() + value = await head_child.read_varstring() head['tags'].append((key, value)) - return head + return cast(GSFFileHeaderDict, head) - def _decode_tils(self, tils_block): + async def _decode_tils(self, tils_block: AsyncGSFBlock) -> List[dict]: """Decode timelabels (tils) block :param tils_block: Instance of GSFBlock() representing a "gbhd" block :returns: tils block as a dict """ tils = [] - timelabel_count = tils_block.read_uint(2) + timelabel_count = await tils_block.read_uint(2) for i in range(0, timelabel_count): - tag = tils_block.read_string(16) + tag = await tils_block.read_string(16) tag = tag.strip("\x00") - count = tils_block.read_uint(4) - rate = tils_block.read_rational() - drop = tils_block.read_bool() + count = await tils_block.read_uint(4) + rate = await tils_block.read_rational() + drop = await tils_block.read_bool() tils.append({'tag': tag, 'timelabel': {'frames_since_midnight': count, @@ -471,46 +583,46 @@ def _decode_tils(self, tils_block): return tils - def _decode_gbhd(self, gbhd_block): + async def _decode_gbhd(self, gbhd_block: AsyncGSFBlock) -> GrainMetadataDict: """Decode grain block header ("gbhd") to get grain metadata :param gbhd_block: Instance of GSFBlock() representing a "gbhd" block :returns: Grain data dict :raises GSFDecodeError: If "gbhd" block contains an unkown child block """ - meta = { + meta: dict = { "grain": { } } - meta['grain']['source_id'] = gbhd_block.read_uuid() - meta['grain']['flow_id'] = gbhd_block.read_uuid() + meta['grain']['source_id'] = await gbhd_block.read_uuid() + meta['grain']['flow_id'] = await gbhd_block.read_uuid() self.file_data.seek(16, 1) # Skip over deprecated byte array - meta['grain']['origin_timestamp'] = gbhd_block.read_ippts() - meta['grain']['sync_timestamp'] = gbhd_block.read_ippts() - meta['grain']['rate'] = gbhd_block.read_rational() - meta['grain']['duration'] = gbhd_block.read_rational() + meta['grain']['origin_timestamp'] = await gbhd_block.read_ippts() + meta['grain']['sync_timestamp'] = await gbhd_block.read_ippts() + meta['grain']['rate'] = await gbhd_block.read_rational() + meta['grain']['duration'] = await gbhd_block.read_rational() - for gbhd_child in gbhd_block.child_blocks(): + async for gbhd_child in gbhd_block.child_blocks(): if gbhd_child.tag == "tils": - meta['grain']['timelabels'] = self._decode_tils(gbhd_child) + meta['grain']['timelabels'] = await self._decode_tils(gbhd_child) elif gbhd_child.tag == "vghd": meta['grain']['grain_type'] = 'video' meta['grain']['cog_frame'] = {} - meta['grain']['cog_frame']['format'] = gbhd_child.read_uint(4) - meta['grain']['cog_frame']['layout'] = gbhd_child.read_uint(4) - meta['grain']['cog_frame']['width'] = gbhd_child.read_uint(4) - meta['grain']['cog_frame']['height'] = gbhd_child.read_uint(4) - meta['grain']['cog_frame']['extension'] = gbhd_child.read_uint(4) + meta['grain']['cog_frame']['format'] = await gbhd_child.read_uint(4) + meta['grain']['cog_frame']['layout'] = await gbhd_child.read_uint(4) + meta['grain']['cog_frame']['width'] = await gbhd_child.read_uint(4) + meta['grain']['cog_frame']['height'] = await gbhd_child.read_uint(4) + meta['grain']['cog_frame']['extension'] = await gbhd_child.read_uint(4) - src_aspect_ratio = gbhd_child.read_rational() + src_aspect_ratio = await gbhd_child.read_rational() if src_aspect_ratio != 0: meta['grain']['cog_frame']['source_aspect_ratio'] = { 'numerator': src_aspect_ratio.numerator, 'denominator': src_aspect_ratio.denominator } - pixel_aspect_ratio = gbhd_child.read_rational() + pixel_aspect_ratio = await gbhd_child.read_rational() if pixel_aspect_ratio != 0: meta['grain']['cog_frame']['pixel_aspect_ratio'] = { 'numerator': pixel_aspect_ratio.numerator, @@ -519,18 +631,18 @@ def _decode_gbhd(self, gbhd_block): meta['grain']['cog_frame']['components'] = [] if gbhd_child.has_child_block(): - with GSFBlock(self.file_data) as comp_block: + async with AsyncGSFBlock(self.file_data) as comp_block: if comp_block.tag != "comp": continue # Skip unknown/unexpected block - comp_count = comp_block.read_uint(2) + comp_count = await comp_block.read_uint(2) offset = 0 for i in range(0, comp_count): comp = {} - comp['width'] = comp_block.read_uint(4) - comp['height'] = comp_block.read_uint(4) - comp['stride'] = comp_block.read_uint(4) - comp['length'] = comp_block.read_uint(4) + comp['width'] = await comp_block.read_uint(4) + comp['height'] = await comp_block.read_uint(4) + comp['stride'] = await comp_block.read_uint(4) + comp['length'] = await comp_block.read_uint(4) comp['offset'] = offset offset += comp['length'] meta['grain']['cog_frame']['components'].append(comp) @@ -538,40 +650,40 @@ def _decode_gbhd(self, gbhd_block): elif gbhd_child.tag == 'cghd': meta['grain']['grain_type'] = "coded_video" meta['grain']['cog_coded_frame'] = {} - meta['grain']['cog_coded_frame']['format'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['layout'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['origin_width'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['origin_height'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['coded_width'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['coded_height'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_frame']['is_key_frame'] = gbhd_child.read_bool() - meta['grain']['cog_coded_frame']['temporal_offset'] = gbhd_child.read_sint(4) + meta['grain']['cog_coded_frame']['format'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['layout'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['origin_width'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['origin_height'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['coded_width'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['coded_height'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_frame']['is_key_frame'] = await gbhd_child.read_bool() + meta['grain']['cog_coded_frame']['temporal_offset'] = await gbhd_child.read_sint(4) if gbhd_child.has_child_block(): - with GSFBlock(self.file_data) as unof_block: + async with AsyncGSFBlock(self.file_data) as unof_block: meta['grain']['cog_coded_frame']['unit_offsets'] = [] - unit_offsets = unof_block.read_uint(2) + unit_offsets = await unof_block.read_uint(2) for i in range(0, unit_offsets): - meta['grain']['cog_coded_frame']['unit_offsets'].append(unof_block.read_uint(4)) + meta['grain']['cog_coded_frame']['unit_offsets'].append(await unof_block.read_uint(4)) elif gbhd_child.tag == "aghd": meta['grain']['grain_type'] = "audio" meta['grain']['cog_audio'] = {} - meta['grain']['cog_audio']['format'] = gbhd_child.read_uint(4) - meta['grain']['cog_audio']['channels'] = gbhd_child.read_uint(2) - meta['grain']['cog_audio']['samples'] = gbhd_child.read_uint(4) - meta['grain']['cog_audio']['sample_rate'] = gbhd_child.read_uint(4) + meta['grain']['cog_audio']['format'] = await gbhd_child.read_uint(4) + meta['grain']['cog_audio']['channels'] = await gbhd_child.read_uint(2) + meta['grain']['cog_audio']['samples'] = await gbhd_child.read_uint(4) + meta['grain']['cog_audio']['sample_rate'] = await gbhd_child.read_uint(4) elif gbhd_child.tag == "cahd": meta['grain']['grain_type'] = "coded_audio" meta['grain']['cog_coded_audio'] = {} - meta['grain']['cog_coded_audio']['format'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_audio']['channels'] = gbhd_child.read_uint(2) - meta['grain']['cog_coded_audio']['samples'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_audio']['priming'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_audio']['remainder'] = gbhd_child.read_uint(4) - meta['grain']['cog_coded_audio']['sample_rate'] = gbhd_child.read_uint(4) + meta['grain']['cog_coded_audio']['format'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_audio']['channels'] = await gbhd_child.read_uint(2) + meta['grain']['cog_coded_audio']['samples'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_audio']['priming'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_audio']['remainder'] = await gbhd_child.read_uint(4) + meta['grain']['cog_coded_audio']['sample_rate'] = await gbhd_child.read_uint(4) elif gbhd_child.tag == "eghd": meta['grain']['grain_type'] = "event" @@ -582,86 +694,287 @@ def _decode_gbhd(self, gbhd_block): length=gbhd_child.size ) - return meta + return cast(GrainMetadataDict, meta) - def decode_file_headers(self): - """Verify the file is a supported version, and get the file header + async def _decode_file_headers(self) -> None: + """Verify the file is a supported version, get the file header and store it in the file_headers property - :returns: File header data (segments and tags) as a dict :raises GSFDecodeBadVersionError: If the file version is not supported :raises GSFDecodeBadFileTypeError: If this isn't a GSF file :raises GSFDecodeError: If the file doesn't have a "head" block """ - (major, minor) = self._decode_ssb_header() + (major, minor) = await self._decode_ssb_header() if (major, minor) != (7, 0): raise GSFDecodeBadVersionError("Unknown Version {}.{}".format(major, minor), 0, major, minor) try: - with GSFBlock(self.file_data, want_tag="head") as head_block: - return self._decode_head(head_block) + async with AsyncGSFBlock(self.file_data, want_tag="head") as head_block: + self.file_headers = await self._decode_head(head_block) except EOFError: raise GSFDecodeError("No head block found in file", self.file_data.tell()) - def grains(self, local_ids=None, skip_data=False, load_lazily=False): + def _add_lazy_grain(self, key: int, grain: GRAIN): + self._unloaded_lazy_grains[key] = grain + + def _remove_lazy_grain(self, key: int): + del self._unloaded_lazy_grains[key] + + async def _kill_unused_lazy_loaders(self): + self._exiting = True + for (key, grain) in self._unloaded_lazy_grains.items(): + await grain + + async def _load_unused_lazy_loaders(self): + for (key, grain) in list(self._unloaded_lazy_grains.items()): + await grain + + async def grains(self, + local_ids: Optional[Sequence[int]] = None, + loading_mode: GrainDataLoadingMode = GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE) -> AsyncIterable[Tuple[GRAIN, int]]: """Generator to get grains from the GSF file. Skips blocks which aren't "grai". The file_data will be positioned after the `grai` block. :param local_ids: A list of local-ids to include in the output. If None (the default) then all local-ids will be included - :param skip_data: If True, grain data blocks will be seeked over and only grain headers will be read - :param load_lazily: If True, the grains returned will be designed to lazily load data from the underlying stream - only when it is needed. In this case the "skip_data" parameter will be ignored. + :param loading_mode: The mode to use when loading grain data elements. For modes ALWAYS_DEFER_LOAD_IF_POSSIBLE and + ALWAYS_LOAD_DEFER_IF_POSSIBLE with a seekable input the grain data can be loaded later by awaiting the + grain object itself. as long as you are still inside this context manager. When the context manager exits + all grains are either implicitly loaded or rendered permanently empty. :yields: (Grain, local_id) tuple for each grain :raises GSFDecodeError: If grain is invalid (e.g. no "gbhd" child) """ + async def _read_out_of_order(parent: GSFAsyncDecoderSession, + key: int, + file_data: OpenAsyncBinaryIO, + pos: int, + length: int, + load_on_exit=False) -> Optional[bytes]: + if load_on_exit or not parent._exiting: + parent._remove_lazy_grain(key) + + oldpos = file_data.tell() + file_data.seek(pos) + data = await file_data.read(length) + file_data.seek(oldpos) + return data + else: + return None + while True: try: - with GSFBlock(self.file_data, want_tag="grai") as grai_block: + async with AsyncGSFBlock(self.file_data, want_tag="grai") as grai_block: if grai_block.size == 0: return # Terminator block reached - local_id = grai_block.read_uint(2) + local_id = await grai_block.read_uint(2) if local_ids is not None and local_id not in local_ids: continue - with GSFBlock(self.file_data, want_tag="gbhd", raise_on_wrong_tag=True) as gbhd_block: - meta = self._decode_gbhd(gbhd_block) + async with AsyncGSFBlock(self.file_data, want_tag="gbhd", raise_on_wrong_tag=True) as gbhd_block: + meta = await self._decode_gbhd(gbhd_block) - data = None + data: Optional[Union[bytes, Awaitable[Optional[bytes]]]] = None + data_length = 0 if grai_block.has_child_block(): - with GSFBlock(self.file_data, want_tag="grdt") as grdt_block: + async with AsyncGSFBlock(self.file_data, want_tag="grdt") as grdt_block: if grdt_block.get_remaining() > 0: - if load_lazily: - data = IOBytes(self.file_data, self.file_data.tell(), grdt_block.get_remaining()) - elif not skip_data: - data = self.file_data.read(grdt_block.get_remaining()) - - yield (self.Grain(meta, data), local_id) + if self.file_data.seekable_backwards() and loading_mode in [GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE, + GrainDataLoadingMode.ALWAYS_LOAD_DEFER_IF_POSSIBLE]: + if not self._sync_compatibility_mode: + # It is correct that this is not awaited here + # It will be awaited when the data is actually needed. + data = _read_out_of_order(self, + self._next_lazy_grain_number, + self.file_data, + self.file_data.tell(), + grdt_block.get_remaining(), + load_on_exit=(loading_mode == GrainDataLoadingMode.ALWAYS_LOAD_DEFER_IF_POSSIBLE)) + data_length = grdt_block.get_remaining() + else: + # This is compatibility mode with the old code + data = IOBytes(cast(OpenAsyncFileWrapper, self.file_data).getsync(), + self.file_data.tell(), + grdt_block.get_remaining()) + elif loading_mode == GrainDataLoadingMode.LOAD_NEVER: + if self.file_data.seekable_forwards(): + self.file_data.seek(grdt_block.get_remaining(), SEEK_CUR) + else: + await self.file_data.read(grdt_block.get_remaining()) + else: + data = await self.file_data.read(grdt_block.get_remaining()) + + grain = self.Grain(meta, data) + + if isawaitable(data): + grain.length = data_length + self._add_lazy_grain(self._next_lazy_grain_number, grain) + self._next_lazy_grain_number += 1 + + yield (grain, local_id) except EOFError: return # We ran out of grains to read and hit EOF - def decode(self, s=None): + +class GSFSyncDecoderSession (Synchronised[GSFAsyncDecoderSession]): + def __init__(self, other: GSFAsyncDecoderSession): + super().__init__(other) + self.file_headers: Optional[GSFFileHeaderDict] + + def grains(self, + local_ids: Optional[Sequence[int]] = None, + loading_mode: GrainDataLoadingMode = GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE) -> Iterable[Tuple[GRAIN, int]]: + return super().__getattr__('grains')(local_ids, loading_mode) + + +class GSFDecoder(object): + """A decoder for GSF format. + + The preferred interface for usage is to use this class as a context manager, which provides an instance of + GSFDecoderSession. + + For backwards compatibility provides methods to decode the header of a GSF file, followed by a generator to + get each grain, wrapped in some grain method (mediagrains.Grain by default.) These methods are deprecated + and should not be used in new code. + + Can also be used to make a one-off decode of a GSF file from a bytes-like object by calling `decode(bytes_like)`. + """ + def __init__(self, + parse_grain: Callable[[GrainMetadataDict, GrainDataParameterType], GRAIN] = Grain, + file_data: Optional[Union[IO[bytes], AsyncBinaryIO, OpenAsyncBinaryIO]] = None, + **kwargs): + """Constructor + + :param parse_grain: Function that takes a (metadata dict, buffer) and returns a grain representation + :param file_data: BufferedReader (or similar) containing GSF data to decode + """ + self._file_data: Optional[Union[RawIOBase, BufferedIOBase]] + self._afile_data: Optional[AsyncBinaryIO] + self._open_afile: Optional[OpenAsyncBinaryIO] + + self.Grain = parse_grain + + if isinstance(file_data, AsyncBinaryIO): + self._afile_data = file_data + self._open_afile = None + elif isinstance(file_data, OpenAsyncBinaryIO): + self._afile_data = None + self._open_afile = file_data + elif isinstance(file_data, (RawIOBase, BufferedIOBase)): + self._afile_data = AsyncFileWrapper(file_data) + self._open_afile = None + else: + self._afile_data = None + self._open_afile = None + + self._open_session: Optional[GSFSyncDecoderSession] = None + self._open_asession: Optional[GSFAsyncDecoderSession] = None + + self._sync_compatibility_mode: bool = False + + def __enter__(self) -> GSFSyncDecoderSession: + self._sync_compatibility_mode = True + a_session = run_awaitable_synchronously(self.__aenter__()) + return GSFSyncDecoderSession(a_session) + + def __exit__(self, *args, **kwargs): + run_awaitable_synchronously(self._open_asession._load_unused_lazy_loaders()) + run_awaitable_synchronously(self.__aexit__(*args, **kwargs)) + self._sync_compatibility_mode = False + + async def __aenter__(self) -> GSFAsyncDecoderSession: + if self._open_afile is None: + if isinstance(self._afile_data, AsyncBinaryIO): + self._open_afile = await self._afile_data.__aenter__() + else: + raise TypeError("file_data must be an asynchronous binary file to use this class as an async context manager") + + self._open_asession = GSFAsyncDecoderSession(file_data=self._open_afile, + parse_grain=self.Grain, + sync_compatibility_mode=self._sync_compatibility_mode) + await self._open_asession._decode_file_headers() + return self._open_asession + + async def __aexit__(self, *args, **kwargs): + if self._open_asession is not None: + await self._open_asession._kill_unused_lazy_loaders() + self._open_asession = None + + if self._open_afile is not None and self._afile_data is not None: + await self._afile_data.__aexit__(*args, **kwargs) + self._open_afile = None + + @deprecated(version="2.7.0", reason="This method is old, use the class as a context manager instead") + def decode_file_headers(self) -> GSFFileHeaderDict: + """Verify the file is a supported version, and get the file header + + :returns: File header data (segments and tags) as a dict + :raises GSFDecodeBadVersionError: If the file version is not supported + :raises GSFDecodeBadFileTypeError: If this isn't a GSF file + :raises GSFDecodeError: If the file doesn't have a "head" block + """ + self._open_session = self.__enter__() + if self._open_session.file_headers is None: + raise RuntimeError("There should be some file headers here!") + return self._open_session.file_headers + + @deprecated(version="2.7.0", reason="This method is old, use the class as a context manager instead") + def grains(self, + local_ids: Optional[Sequence[int]] = None, + skip_data: bool = False, + load_lazily: bool = False): + """Generator to get grains from the GSF file. Skips blocks which aren't "grai". + + The file_data will be positioned after the `grai` block. + + :param local_ids: A list of local-ids to include in the output. If None (the default) then all local-ids will be + included + :param skip_data: If True, grain data blocks will be seeked over and only grain headers will be read + :param load_lazily: If True, the grains returned will be designed to lazily load data from the underlying stream + only when it is needed. In this case the "skip_data" parameter will be ignored. + :yields: (Grain, local_id) tuple for each grain + :raises GSFDecodeError: If grain is invalid (e.g. no "gbhd" child) + """ + if self._open_session is None: + raise RuntimeError("Cannot access grains when no headers have been decoded") + + if load_lazily: + mode = GrainDataLoadingMode.ALWAYS_LOAD_DEFER_IF_POSSIBLE + elif skip_data: + mode = GrainDataLoadingMode.LOAD_NEVER + else: + mode = GrainDataLoadingMode.LOAD_IMMEDIATELY + + return self._open_session.grains(local_ids=local_ids, loading_mode=mode) + + async def _asynchronously_decode(self) -> Tuple[GSFFileHeaderDict, Dict[int, List[GRAIN]]]: + async with self as dec: + grains: Dict[int, List[GRAIN]] = {} + async for (grain, key) in dec.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY): + if key not in grains: + grains[key] = [] + grains[key].append(grain) + if dec.file_headers is None: + raise RuntimeError("There ought to be file headers here") + return (dec.file_headers, grains) + + def decode(self, s: Optional[bytes] = None) -> Tuple[GSFFileHeaderDict, Dict[int, List[GRAIN]]]: """Decode a GSF formatted bytes object :param s: GSF-formatted bytes object, optional if `file_data` supplied to constructor :returns: A dictionary mapping sequence ids to lists of GRAIN objects (or subclasses of such). """ if (s is not None): - self.file_data = BytesIO(s) + # Unclear why this cast is needed, since a BytesIO is already a BufferedIOBase ... + self._file_data = cast(BufferedIOBase, BytesIO(s)) - head = self.decode_file_headers() + rval = run_awaitable_synchronously(self._asynchronously_decode()) - segments = {} - - for (grain, local_id) in self.grains(): - if local_id not in segments: - segments[local_id] = [] - segments[local_id].append(grain) - - return (head, segments) + if rval is None: + raise RuntimeError("Running asynchronous decode synchronously returned nothing") + return rval class GSFEncodeError(GSFError): @@ -675,45 +988,329 @@ class GSFEncodeAddToActiveDump(GSFEncodeError): pass -def _write_uint(file, val, size): +def _encode_uint(val: int, size: int) -> bytes: d = bytearray(size) for i in range(0, size): d[i] = (val & 0xFF) val >>= 8 - file.write(d) + return bytes(d) -def _write_sint(file, val, size): +def _encode_sint(val: int, size: int) -> bytes: if val < 0: val = val + (1 << (8*size)) - _write_uint(file, val, size) + return _encode_uint(val, size) -def _write_uuid(file, val): - file.write(val.bytes) +def _encode_uuid(val: UUID) -> bytes: + return val.bytes -def _write_ts(file, ts): - _write_uint(file, ts.sec, 6) - _write_uint(file, ts.ns, 4) +def _encode_ts(ts: Timestamp) -> bytes: + return (_encode_uint(ts.sec, 6) + + _encode_uint(ts.ns, 4)) -def _write_rational(file, value): +def _encode_rational(value: RationalTypes) -> bytes: value = Fraction(value) - _write_uint(file, value.numerator, 4) - _write_uint(file, value.denominator, 4) + return (_encode_uint(value.numerator, 4) + + _encode_uint(value.denominator, 4)) + + +class OpenGSFEncoderBase(object): + def __init__(self, + major: int, + minor: int, + id: UUID, + created: datetime, + tags: List["GSFEncoderTag"], + segments: Dict[int, "GSFEncoderSegment"], + streaming: bool, + next_local: int): + self.major = major + self.minor = minor + self._tags = tags + self.streaming = streaming + self.id = id + self.created = created + self._segments = segments + self._next_local = next_local + self._active_dump = False + @property + def tags(self) -> Tuple["GSFEncoderTag", ...]: + return tuple(self._tags) -def seekable(file): # pragma: no cover - if PY3: - return file.seekable() - else: - try: - file.tell() - except IOError: - return False + @property + def segments(self) -> Mapping[int, "GSFEncoderSegment"]: + return frozendict(self._segments) + + def add_tag(self, key: str, value: str): + """Add a tag to the file""" + if self._active_dump: + raise GSFEncodeAddToActiveDump("Cannot add a new tag to an encoder that is currently dumping") + + self._tags.append(GSFEncoderTag(key, value)) + + def add_segment(self, id: Optional[UUID] = None, local_id: Optional[int] = None, tags: Optional[Iterable[Tuple[str, str]]] = None) -> "GSFEncoderSegment": + """Add a segment to the file, if id is specified it should be a uuid, + otherwise one will be generated. If local_id is specified it should be an + integer, otherwise the next available integer will be used. Returns the newly + created segment.""" + + if local_id is None: + local_id = self._next_local + if local_id >= self._next_local: + self._next_local = local_id + 1 + if local_id in self._segments: + raise GSFEncodeError("Segment local id {} already in use".format(local_id)) + + if id is None: + id = uuid1() + + if self._active_dump: + raise GSFEncodeAddToActiveDump("Cannot add a new segment {} ({!s}) to an encoder that is currently dumping".format(local_id, id)) + + seg = GSFEncoderSegment(id, local_id, tags=tags) + self._segments[local_id] = seg + return seg + + def _get_segment(self, segment_id: Optional[UUID], segment_local_id: Optional[int]) -> "GSFEncoderSegment": + if segment_local_id is None: + segments = sorted([local_id for local_id in self._segments if segment_id is None or self._segments[local_id].id == segment_id]) + if len(segments) > 0: + segment_local_id = segments[0] + if segment_local_id is not None and segment_local_id in self._segments: + segment = self._segments[segment_local_id] else: - return True + if self._active_dump: + raise GSFEncodeError("Cannot add a segment to a progressive dump") + segment = self.add_segment(id=segment_id, local_id=segment_local_id) + + return segment + + def _set_segment_offsets(self, segment_offsets: Iterable[Tuple["GSFEncoderSegment", int]], pos: int) -> None: + for (seg, offset) in segment_offsets: + seg.set_size_position(pos + offset) + + def _encode_file_header(self): + return (b"SSBB" + + b"grsg" + + _encode_uint(self.major, 2) + + _encode_uint(self.minor, 2)) + + def _encode_head_block(self, all_at_once: bool = False) -> Tuple[bytes, List[Tuple["GSFEncoderSegment", int]]]: + size = (31 + + sum(seg.segm_block_size for seg in self._segments.values()) + + sum(tag.tag_block_size for tag in self._tags)) + + data = ( + b"head" + + _encode_uint(size, 4) + + _encode_uuid(self.id) + + _encode_sint(self.created.year, 2) + + _encode_uint(self.created.month, 1) + + _encode_uint(self.created.day, 1) + + _encode_uint(self.created.hour, 1) + + _encode_uint(self.created.minute, 1) + + _encode_uint(self.created.second, 1)) + offsets = [] + + for seg in self._segments.values(): + (seg_data, offset) = seg._encode_header(all_at_once=all_at_once) + offsets.append((seg, len(data) + offset)) + data += seg_data + + for tag in self._tags: + data += bytes(tag) + + return (data, offsets) + + def _encode_all_grains(self): + data = b'' + for seg in self._segments.values(): + data += seg.encode_all_grains() + return data + + +class OpenGSFEncoder(OpenGSFEncoderBase): + def __init__(self, + file: IO[bytes], + major: int, + minor: int, + id: UUID, + created: datetime, + tags: List["GSFEncoderTag"], + segments: Dict[int, "GSFEncoderSegment"], + streaming: bool, + next_local: int): + super().__init__(major, minor, id, created, tags, segments, streaming, next_local) + self.file = file + + def add_grain(self, + grain: GRAIN, + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): + """Add a grain to one of the segments of the file. If no local_segment_id + is provided then a segment with id equal to segment_id will be used if one + exists, or the lowest numeric segmemnt if segment_id was not provided. + + If no segment matching the criteria exists then one will be created. + """ + self.add_grains((grain,), segment_id=segment_id, segment_local_id=segment_local_id) + + def add_grains(self, + grains: Iterable[GRAIN], + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): + """Add several grains to one of the segments of the file. If no local_segment_id + is provided then a segment with id equal to segment_id will be used if one + exists, or the lowest numeric segmemnt if segment_id was not provided. + + If no segment matching the criteria exists then one will be created. + """ + segment = self._get_segment(segment_id, segment_local_id) + + if self._active_dump: + for grain in grains: + self.file.write(segment.encode_grain(grain)) + else: + segment.add_grains(grains) + + def _truncate(self): + if self.file.seekable(): + self.file.seek(0) + self.file.truncate() + + def _start_dump(self, all_at_once: bool = False): + self._active_dump = True + + self._truncate() + + file_header = self._encode_file_header() + + (head_block, segment_offsets) = self._encode_head_block(all_at_once=all_at_once) + if not all_at_once and self.file.seekable(): + self._set_segment_offsets(segment_offsets, self.file.tell() + len(file_header)) + + self.file.write(file_header + + head_block + + self._encode_all_grains()) + + def _end_dump(self): + for seg in self._segments.values(): + if self.file.seekable() and seg._count_pos != -1: + curpos = self.file.tell() + self.file.seek(seg._count_pos) + self.file.write(_encode_sint(seg.get_write_count(), 8)) + self.file.seek(curpos) + + if self._active_dump: + self.file.write(b"grai" + + _encode_uint(0, 4)) + self._active_dump = False + + +class OpenAsyncGSFEncoder(OpenGSFEncoderBase): + def __init__(self, + file: Union[AsyncBinaryIO, OpenAsyncBinaryIO], + major: int, + minor: int, + id: UUID, + created: datetime, + tags: List["GSFEncoderTag"], + segments: Dict[int, "GSFEncoderSegment"], + streaming: bool, + next_local: int): + super().__init__(major, minor, id, created, tags, segments, streaming, next_local) + self.file: Optional[AsyncBinaryIO] + self._open_file: Optional[OpenAsyncBinaryIO] + + if isinstance(file, AsyncBinaryIO): + self.file = file + self._open_file = None + else: + self.file = None + self._open_file = file + + async def add_grain(self, + grain: GRAIN, + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): + """Add a grain to one of the segments of the file. If no local_segment_id + is provided then a segment with id equal to segment_id will be used if one + exists, or the lowest numeric segmemnt if segment_id was not provided. + + If no segment matching the criteria exists then one will be created. + """ + await self.add_grains((grain,), segment_id=segment_id, segment_local_id=segment_local_id) + + async def add_grains(self, + grains: Iterable[GRAIN], + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): + """Add several grains to one of the segments of the file. If no local_segment_id + is provided then a segment with id equal to segment_id will be used if one + exists, or the lowest numeric segmemnt if segment_id was not provided. + + If no segment matching the criteria exists then one will be created. + """ + segment = self._get_segment(segment_id, segment_local_id) + + if self._open_file is not None and self._active_dump: + for grain in grains: + await self._open_file.write(segment.encode_grain(grain)) + else: + segment.add_grains(grains) + + async def _truncate(self) -> None: + if self._open_file is not None and self._open_file.seekable(): + self._open_file.seek(0) + await self._open_file.truncate() + + async def _start_dump(self, all_at_once: bool = False): + self._active_dump = True + + if self._open_file is None: + if self.file is not None: + self._open_file = await self.file.__aenter__() + else: + raise GSFEncodeError("Tried to encode to a file without a file") + + await self._truncate() + + file_header = self._encode_file_header() + + (head_block, segment_offsets) = self._encode_head_block(all_at_once=all_at_once) + + if not all_at_once and self._open_file.seekable(): + self._set_segment_offsets(segment_offsets, self._open_file.tell() + len(file_header)) + + await self._open_file.write(file_header + + head_block + + self._encode_all_grains()) + + async def _end_dump(self): + for seg in self._segments.values(): + if self._open_file.seekable() and seg._count_pos != -1: + curpos = self._open_file.tell() + self._open_file.seek(seg._count_pos) + + await self._open_file.write(_encode_sint(seg.get_write_count(), 8)) + + self._open_file.seek(curpos) + + if self._active_dump: + await self._open_file.write(b"grai" + + _encode_uint(0, 4)) + + self._active_dump = False + + +class SegmentDict(TypedDict, total=False): + id: UUID + local_id: int + tags: Iterable[Tuple[str, str]] class GSFEncoder(object): @@ -723,8 +1320,14 @@ class GSFEncoder(object): optional arguments exist for specifying file-level metadata, if no created time is specified the current time will be used, if no id is specified one will be generated randomly. - The main interface are the methods add_grain and dump which add a grain to the file and dump the file to - the buffer respectively. + + The recommended interface is to use the encoder as either a context manager or an asynchronous context + manager. Whilst in the context manager new grains can be added with add_grain, and upon leaving the context + manager the grains will be written to the file. If the `streaming=True` parameter is passed to the constructor + then calls to add_grain within the context manager will instead cause the grain to be written immediately. + + And older deprecated interface exists for synchronous work: the method add_grain and dump which add a grain to + the file and dump the file the the buffer respectively. If a streaming format is required then you can instead use the "start_dump" method, followed by adding grains as needed, and then the "end_dump" method. Each new grain will be written as it is added. In this mode @@ -742,21 +1345,42 @@ class GSFEncoder(object): The current version of the library is designed for compatibility with v.7.0 of the GSF format. Setting a different version number will simply change the reported version number in the file, but will not alter the syntax at all. If future versions of this code add support for other versions of GSF then this will change.""" - def __init__(self, file, major=7, minor=0, id=None, created=None, tags=None): + def __init__(self, + file: Union[IO[bytes], AsyncBinaryIO, OpenAsyncBinaryIO], + major: int = 7, + minor: int = 0, + id: Optional[UUID] = None, + created: Optional[datetime] = None, + tags: Iterable[Tuple[str, str]] = None, + segments: Iterable[SegmentDict] = [], + streaming: bool = False): self.file = file self.major = major self.minor = minor - self.id = id - self.created = created - self._tags = [] + self._tags: List["GSFEncoderTag"] = [] + self.streaming = streaming + self._open_encoder: Optional[OpenGSFEncoder] = None + self._open_async_encoder: Optional[OpenAsyncGSFEncoder] = None + self._next_local = 1 - if self.id is None: + if id is None: self.id = uuid1() - if self.created is None: + else: + self.id = id + + if created is None: self.created = datetime.now() - self._segments = {} - self._next_local = 1 - self._active_dump = False + else: + self.created = created + + self._segments: Dict[int, "GSFEncoderSegment"] = {} + + if segments is not None: + for seg in segments: + try: + self.add_segment(**seg) + except (TypeError, IndexError): + raise GSFEncodeError("No idea how to turn {!r} into a segment".format(seg)) if tags is not None: for tag in tags: @@ -765,27 +1389,78 @@ def __init__(self, file, major=7, minor=0, id=None, created=None, tags=None): except (TypeError, IndexError): raise GSFEncodeError("No idea how to turn {!r} into a tag".format(tag)) + def __enter__(self) -> OpenGSFEncoder: + if not isinstance(self.file, RawIOBase) and not isinstance(self.file, BufferedIOBase): + raise ValueError("To use in synchronous mode the file must be a synchronously writeable file") + self._open_encoder = OpenGSFEncoder(self.file, + self.major, + self.minor, + self.id, + self.created, + self._tags, + self._segments, + self.streaming, + self._next_local) + if self.streaming: + self._open_encoder._start_dump(all_at_once=False) + return self._open_encoder + + def __exit__(self, *args, **kwargs): + if self._open_encoder is not None: + if not self.streaming: + self._open_encoder._start_dump(all_at_once=True) + self._open_encoder._end_dump() + self._next_local = self._open_encoder._next_local + self._open_encoder = None + + async def __aenter__(self): + if not isinstance(self.file, AsyncBinaryIO) and not isinstance(self.file, OpenAsyncBinaryIO): + raise ValueError("To use in asynchronous mode the file must be an asynchronously writeable file-like object") + self._open_async_encoder = OpenAsyncGSFEncoder(self.file, + self.major, + self.minor, + self.id, + self.created, + self._tags, + self._segments, + self.streaming, + self._next_local) + if self.streaming: + await self._open_async_encoder._start_dump(all_at_once=False) + return self._open_async_encoder + + async def __aexit__(self, *args, **kwargs): + if self._open_async_encoder is not None: + if not self.streaming: + await self._open_async_encoder._start_dump(all_at_once=True) + await self._open_async_encoder._end_dump() + self._next_local = self._open_async_encoder._next_local + self._open_async_encoder = None + @property - def tags(self): + def tags(self) -> Tuple["GSFEncoderTag", ...]: return tuple(self._tags) @property - def segments(self): + def segments(self) -> Mapping[int, "GSFEncoderSegment"]: return frozendict(self._segments) - def add_tag(self, key, value): + def add_tag(self, key: str, value: str): """Add a tag to the file""" - if self._active_dump: + if self._open_encoder is not None: raise GSFEncodeAddToActiveDump("Cannot add a new tag to an encoder that is currently dumping") self._tags.append(GSFEncoderTag(key, value)) - def add_segment(self, id=None, local_id=None, tags=None): + def add_segment(self, id: Optional[UUID] = None, local_id: Optional[int] = None, tags: Optional[Iterable[Tuple[str, str]]] = None) -> "GSFEncoderSegment": """Add a segment to the file, if id is specified it should be a uuid, otherwise one will be generated. If local_id is specified it should be an integer, otherwise the next available integer will be used. Returns the newly created segment.""" + if self._open_encoder is not None: + raise GSFEncodeAddToActiveDump("Cannot add a new segment {} ({!s}) to an encoder that is currently dumping".format(local_id, id)) + if local_id is None: local_id = self._next_local if local_id >= self._next_local: @@ -796,14 +1471,14 @@ def add_segment(self, id=None, local_id=None, tags=None): if id is None: id = uuid1() - if self._active_dump: - raise GSFEncodeAddToActiveDump("Cannot add a new segment {} ({!s}) to an encoder that is currently dumping".format(local_id, id)) - seg = GSFEncoderSegment(id, local_id, tags=tags) self._segments[local_id] = seg return seg - def add_grain(self, grain, segment_id=None, segment_local_id=None): + def add_grain(self, + grain: GRAIN, + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): """Add a grain to one of the segments of the file. If no local_segment_id is provided then a segment with id equal to segment_id will be used if one exists, or the lowest numeric segmemnt if segment_id was not provided. @@ -812,86 +1487,60 @@ def add_grain(self, grain, segment_id=None, segment_local_id=None): """ self.add_grains((grain,), segment_id=segment_id, segment_local_id=segment_local_id) - def add_grains(self, grains, segment_id=None, segment_local_id=None): + def add_grains(self, + grains: Iterable[GRAIN], + segment_id: Optional[UUID] = None, + segment_local_id: Optional[int] = None): """Add several grains to one of the segments of the file. If no local_segment_id is provided then a segment with id equal to segment_id will be used if one exists, or the lowest numeric segmemnt if segment_id was not provided. If no segment matching the criteria exists then one will be created. """ - if segment_local_id is None: - segments = sorted([local_id for local_id in self._segments if segment_id is None or self._segments[local_id].id == segment_id]) - if len(segments) > 0: - segment_local_id = segments[0] - if segment_local_id is not None and segment_local_id in self._segments: - segment = self._segments[segment_local_id] + if self._open_encoder is not None: + self._open_encoder.add_grains(grains, segment_id, segment_local_id) else: - segment = self.add_segment(id=segment_id, local_id=segment_local_id) - segment.add_grains(grains) + if segment_local_id is None: + segments = sorted([local_id for local_id in self._segments if segment_id is None or self._segments[local_id].id == segment_id]) + if len(segments) > 0: + segment_local_id = segments[0] + if segment_local_id is not None and segment_local_id in self._segments: + segment = self._segments[segment_local_id] + else: + segment = self.add_segment(id=segment_id, local_id=segment_local_id) + segment.add_grains(grains) + @deprecated(version="2.7.0", reason="This mechanism is deprecated, use a context manager instead") def dump(self): """Dump the whole contents of this encoder to the file in one go, replacing anything that's already there.""" + with self: + pass - self.start_dump(all_at_once=True) - self.end_dump() - + @deprecated(version="2.7.0", reason="This mechanism is deprecated, use a context manager instead") def start_dump(self, all_at_once=False): """Start dumping the contents of this encoder to the specified file, if the file is seakable then it will replace the current content, otherwise it will append.""" - self._active_dump = True - - if seekable(self.file): - self.file.seek(0) - self.file.truncate() - - self._write_file_header() - self._write_head_block(all_at_once=all_at_once) - self._write_all_grains() - + self._open_encoder = OpenGSFEncoder(self.file, + self.major, + self.minor, + self.id, + self.created, + self._tags, + self._segments, + self.streaming, + self._next_local) + self._open_encoder._start_dump(all_at_once=all_at_once) + + @deprecated(version="2.7.0", reason="This mechanism is deprecated, use a context manager instead") def end_dump(self, all_at_once=False): """End the current dump to the file. In a seakable stream this will write all segment counts, in a non-seakable stream it will not.""" - - for seg in self._segments.values(): - seg.complete_write() - - if self._active_dump: - self.file.write(b"grai") - _write_uint(self.file, 0, 4) - self._active_dump = False - - def _write_file_header(self): - self.file.write(b"SSBB") # signature - self.file.write(b"grsg") # file type - _write_uint(self.file, self.major, 2) - _write_uint(self.file, self.minor, 2) - - def _write_head_block(self, all_at_once=False): - size = (31 + - sum(seg.segm_block_size for seg in self._segments.values()) + - sum(tag.tag_block_size for tag in self._tags)) - - self.file.write(b"head") - _write_uint(self.file, size, 4) - _write_uuid(self.file, self.id) - _write_sint(self.file, self.created.year, 2) - _write_uint(self.file, self.created.month, 1) - _write_uint(self.file, self.created.day, 1) - _write_uint(self.file, self.created.hour, 1) - _write_uint(self.file, self.created.minute, 1) - _write_uint(self.file, self.created.second, 1) - - for seg in self._segments.values(): - seg.write_to(self.file, all_at_once=all_at_once) - - for tag in self._tags: - tag.write_to(self.file) - - def _write_all_grains(self): - for seg in self._segments.values(): - seg.write_all_grains() + if self._open_encoder is not None: + self._open_encoder._end_dump() + self._next_local = self._open_encoder._next_local + self._open_encoder = None class GSFEncoderTag(object): @@ -904,45 +1553,46 @@ class GSFEncoderTag(object): both strings.""" - def __init__(self, key, value): + def __init__(self, key: str, value: str): self.key = key self.value = value @property - def encoded_key(self): + def encoded_key(self) -> bytes: return self.key.encode("utf-8")[:65535] @property - def encoded_value(self): + def encoded_value(self) -> bytes: return self.value.encode("utf-8")[:65535] @property - def tag_block_size(self): + def tag_block_size(self) -> int: return 12 + len(self.encoded_key) + len(self.encoded_value) - def write_to(self, file): - file.write(b"tag ") - _write_uint(file, self.tag_block_size, 4) - _write_uint(file, len(self.encoded_key), 2) - file.write(self.encoded_key) - _write_uint(file, len(self.encoded_value), 2) - file.write(self.encoded_value) + def __bytes__(self): + return ( + b"tag " + + _encode_uint(self.tag_block_size, 4) + + _encode_uint(len(self.encoded_key), 2) + + self.encoded_key + + _encode_uint(len(self.encoded_value), 2) + + self.encoded_value) - def __eq__(self, other): - return other.__eq__((self.key, self.value)) + def __eq__(self, other: object) -> bool: + return other == (self.key, self.value) class GSFEncoderSegment(object): """A class to represent a segment within a GSF file, used for constructing them.""" - def __init__(self, id, local_id, tags=None): + def __init__(self, id: UUID, local_id: int, tags: Iterable[Tuple[str, str]] = None): self.id = id self.local_id = local_id self._write_count = 0 self._count_pos = -1 - self._file = None - self._tags = [] - self._grains = [] + self._active_dump: bool = False + self._tags: List[GSFEncoderTag] = [] + self._grains: List[GRAIN] = [] if tags is not None: for tag in tags: @@ -952,237 +1602,240 @@ def __init__(self, id, local_id, tags=None): raise GSFEncodeError("No idea how to turn {!r} into a tag".format(tag)) @property - def count(self): + def count(self) -> int: return len(self._grains) + self._write_count @property - def segm_block_size(self): + def segm_block_size(self) -> int: return 34 + sum(tag.tag_block_size for tag in self._tags) @property - def tags(self): + def tags(self) -> Tuple[GSFEncoderTag, ...]: return tuple(self._tags) - def write_to(self, file, all_at_once=False): - self._file = file - file.write(b"segm") - _write_uint(file, self.segm_block_size, 4) + def get_write_count(self) -> int: + return self._write_count + + def _encode_header(self, all_at_once: bool = False) -> Tuple[bytes, int]: + self._active_dump = True + data = ( + b"segm" + + _encode_uint(self.segm_block_size, 4) + - _write_uint(file, self.local_id, 2) - _write_uuid(file, self.id) + _encode_uint(self.local_id, 2) + + _encode_uuid(self.id)) + count_pos = len(data) if all_at_once: - _write_sint(file, self.count, 8) + data += _encode_sint(self.count, 8) else: - if seekable(file): - self._count_pos = file.tell() - _write_sint(file, -1, 8) + data += _encode_sint(-1, 8) for tag in self._tags: - tag.write_to(file) + data += bytes(tag) + + return (data, count_pos) + + def set_size_position(self, pos: int): + self._count_pos = pos - def write_all_grains(self): + def encode_all_grains(self) -> bytes: + data = b'' for grain in self._grains: - self._write_grain(grain) + data += self.encode_grain(grain) + self._grains = [] + return data - def _write_grain(self, grain): + def encode_grain(self, grain: GRAIN) -> bytes: gbhd_size = self._gbhd_size_for_grain(grain) - self._file.write(b"grai") - _write_uint(self._file, 10 + gbhd_size + 8 + grain.length, 4) - - _write_uint(self._file, self.local_id, 2) + data = ( + b"grai" + + _encode_uint(10 + gbhd_size + 8 + grain.length, 4) + + _encode_uint(self.local_id, 2) + - self._file.write(b"gbhd") - _write_uint(self._file, gbhd_size, 4) + b"gbhd" + + _encode_uint(gbhd_size, 4) + - _write_uuid(self._file, grain.source_id) - _write_uuid(self._file, grain.flow_id) - self._file.write(b"\x00"*16) - _write_ts(self._file, grain.origin_timestamp) - _write_ts(self._file, grain.sync_timestamp) - _write_rational(self._file, grain.rate) - _write_rational(self._file, grain.duration) + _encode_uuid(grain.source_id) + + _encode_uuid(grain.flow_id) + + b"\x00"*16 + + _encode_ts(grain.origin_timestamp) + + _encode_ts(grain.sync_timestamp) + + _encode_rational(grain.rate) + + _encode_rational(grain.duration)) if len(grain.timelabels) > 0: - self._file.write(b"tils") - _write_uint(self._file, 10 + 29*len(grain.timelabels), 4) + data += ( + b"tils" + + _encode_uint(10 + 29*len(grain.timelabels), 4) + - _write_uint(self._file, len(grain.timelabels), 2) + _encode_uint(len(grain.timelabels), 2)) for label in grain.timelabels: tag = (label['tag'].encode('utf-8') + (b"\x00" * 16))[:16] - self._file.write(tag) - _write_uint(self._file, label['timelabel']['frames_since_midnight'], 4) - _write_uint(self._file, label['timelabel']['frame_rate_numerator'], 4) - _write_uint(self._file, label['timelabel']['frame_rate_denominator'], 4) - _write_uint(self._file, 1 if label['timelabel']['drop_frame'] else 0, 1) + data += ( + tag + + _encode_uint(label['timelabel']['frames_since_midnight'], 4) + + _encode_uint(label['timelabel']['frame_rate_numerator'], 4) + + _encode_uint(label['timelabel']['frame_rate_denominator'], 4) + + _encode_uint(1 if label['timelabel']['drop_frame'] else 0, 1)) if grain.grain_type == "video": - self._write_vghd_for_grain(grain) + data += self._encode_vghd_for_grain(cast(VIDEOGRAIN, grain)) elif grain.grain_type == "coded_video": - self._write_cghd_for_grain(grain) + data += self._encode_cghd_for_grain(cast(CODEDVIDEOGRAIN, grain)) elif grain.grain_type == "audio": - self._write_aghd_for_grain(grain) + data += self._encode_aghd_for_grain(cast(AUDIOGRAIN, grain)) elif grain.grain_type == "coded_audio": - self._write_cahd_for_grain(grain) + data += self._encode_cahd_for_grain(cast(CODEDAUDIOGRAIN, grain)) elif grain.grain_type == "event": - self._write_eghd_for_grain(grain) + data += self._encode_eghd_for_grain(cast(EVENTGRAIN, grain)) elif grain.grain_type != "empty": # pragma: no cover (should be unreachable) raise GSFEncodeError("Unknown grain type: {}".format(grain.grain_type)) - self._file.write(b"grdt") - _write_uint(self._file, 8 + grain.length, 4) + data += ( + b"grdt" + + _encode_uint(8 + grain.length, 4)) if grain.data is not None: - self._file.write(grain.data) + data += bytes(grain.data) self._write_count += 1 - def _gbhd_size_for_grain(self, grain): + return data + + def _gbhd_size_for_grain(self, grain: GRAIN) -> int: size = 92 if len(grain.timelabels) > 0: size += 10 + 29*len(grain.timelabels) if grain.grain_type == "video": - size += self._vghd_size_for_grain(grain) + size += self._vghd_size_for_grain(cast(VIDEOGRAIN, grain)) elif grain.grain_type == "coded_video": - size += self._cghd_size_for_grain(grain) + size += self._cghd_size_for_grain(cast(CODEDVIDEOGRAIN, grain)) elif grain.grain_type == "audio": - size += self._aghd_size_for_grain(grain) + size += self._aghd_size_for_grain(cast(AUDIOGRAIN, grain)) elif grain.grain_type == "coded_audio": - size += self._cahd_size_for_grain(grain) + size += self._cahd_size_for_grain(cast(CODEDAUDIOGRAIN, grain)) elif grain.grain_type == "event": - size += self._eghd_size_for_grain(grain) + size += self._eghd_size_for_grain(cast(EVENTGRAIN, grain)) elif grain.grain_type != "empty": raise GSFEncodeError("Unknown grain type: {}".format(grain.grain_type)) return size - def _vghd_size_for_grain(self, grain): + def _vghd_size_for_grain(self, grain: VIDEOGRAIN) -> int: size = 44 if len(grain.components) > 0: size += 10 + 16*len(grain.components) return size - def _write_vghd_for_grain(self, grain): - self._file.write(b"vghd") - _write_uint(self._file, self._vghd_size_for_grain(grain), 4) + def _encode_vghd_for_grain(self, grain: VIDEOGRAIN) -> bytes: + data = (b"vghd" + + _encode_uint(self._vghd_size_for_grain(grain), 4) + + + _encode_uint(int(grain.format), 4) + + _encode_uint(int(grain.layout), 4) + + _encode_uint(int(grain.width), 4) + + _encode_uint(int(grain.height), 4) + + _encode_uint(int(grain.extension), 4)) - _write_uint(self._file, int(grain.format), 4) - _write_uint(self._file, int(grain.layout), 4) - _write_uint(self._file, int(grain.width), 4) - _write_uint(self._file, int(grain.height), 4) - _write_uint(self._file, int(grain.extension), 4) if grain.source_aspect_ratio is None: - _write_rational(self._file, Fraction(0, 1)) + data += _encode_rational(Fraction(0, 1)) else: - _write_rational(self._file, grain.source_aspect_ratio) + data += _encode_rational(grain.source_aspect_ratio) if grain.pixel_aspect_ratio is None: - _write_rational(self._file, Fraction(0, 1)) + data += _encode_rational(Fraction(0, 1)) else: - _write_rational(self._file, grain.pixel_aspect_ratio) + data += _encode_rational(grain.pixel_aspect_ratio) if len(grain.components) > 0: - self._file.write(b"comp") - _write_uint(self._file, 10 + 16*len(grain.components), 4) + data += (b"comp" + + _encode_uint(10 + 16*len(grain.components), 4) + - _write_uint(self._file, len(grain.components), 2) + _encode_uint(len(grain.components), 2)) for comp in grain.components: - _write_uint(self._file, comp.width, 4) - _write_uint(self._file, comp.height, 4) - _write_uint(self._file, comp.stride, 4) - _write_uint(self._file, comp.length, 4) + data += (_encode_uint(comp.width, 4) + + _encode_uint(comp.height, 4) + + _encode_uint(comp.stride, 4) + + _encode_uint(comp.length, 4)) - def _eghd_size_for_grain(self, grain): - return 9 + return data - def _write_eghd_for_grain(self, grain): - self._file.write(b"eghd") - _write_uint(self._file, self._eghd_size_for_grain(grain), 4) + def _eghd_size_for_grain(self, grain: EVENTGRAIN) -> int: + return 9 - _write_uint(self._file, 0x00, 1) + def _encode_eghd_for_grain(self, grain: EVENTGRAIN) -> bytes: + return (b"eghd" + + _encode_uint(self._eghd_size_for_grain(grain), 4) + + _encode_uint(0x00, 1)) - def _aghd_size_for_grain(self, grain): + def _aghd_size_for_grain(self, grain: AUDIOGRAIN) -> int: return 22 - def _write_aghd_for_grain(self, grain): - self._file.write(b"aghd") - _write_uint(self._file, self._aghd_size_for_grain(grain), 4) + def _encode_aghd_for_grain(self, grain: AUDIOGRAIN) -> bytes: + return (b"aghd" + + _encode_uint(self._aghd_size_for_grain(grain), 4) + - _write_uint(self._file, int(grain.format), 4) - _write_uint(self._file, int(grain.channels), 2) - _write_uint(self._file, int(grain.samples), 4) - _write_uint(self._file, int(grain.sample_rate), 4) + _encode_uint(int(grain.format), 4) + + _encode_uint(int(grain.channels), 2) + + _encode_uint(int(grain.samples), 4) + + _encode_uint(int(grain.sample_rate), 4)) - def _cghd_size_for_grain(self, grain): + def _cghd_size_for_grain(self, grain: CODEDVIDEOGRAIN) -> int: size = 37 if len(grain.unit_offsets) > 0: size += 10 + 4*len(grain.unit_offsets) return size - def _write_cghd_for_grain(self, grain): - self._file.write(b"cghd") - _write_uint(self._file, self._cghd_size_for_grain(grain), 4) + def _encode_cghd_for_grain(self, grain: CODEDVIDEOGRAIN) -> bytes: + data = (b"cghd" + + _encode_uint(self._cghd_size_for_grain(grain), 4) + - _write_uint(self._file, int(grain.format), 4) - _write_uint(self._file, int(grain.layout), 4) - _write_uint(self._file, int(grain.origin_width), 4) - _write_uint(self._file, int(grain.origin_height), 4) - _write_uint(self._file, int(grain.coded_width), 4) - _write_uint(self._file, int(grain.coded_height), 4) - _write_uint(self._file, 1 if grain.is_key_frame else 0, 1) - _write_uint(self._file, int(grain.temporal_offset), 4) + _encode_uint(int(grain.format), 4) + + _encode_uint(int(grain.layout), 4) + + _encode_uint(int(grain.origin_width), 4) + + _encode_uint(int(grain.origin_height), 4) + + _encode_uint(int(grain.coded_width), 4) + + _encode_uint(int(grain.coded_height), 4) + + _encode_uint(1 if grain.is_key_frame else 0, 1) + + _encode_uint(int(grain.temporal_offset), 4)) if len(grain.unit_offsets) > 0: - self._file.write(b"unof") - _write_uint(self._file, 10 + 4*len(grain.unit_offsets), 4) - - _write_uint(self._file, len(grain.unit_offsets), 2) + data += (b"unof" + + _encode_uint(10 + 4*len(grain.unit_offsets), 4) + + _encode_uint(len(grain.unit_offsets), 2)) for i in range(0, len(grain.unit_offsets)): - _write_uint(self._file, grain.unit_offsets[i], 4) - - def _cahd_size_for_grain(self, grain): - return 30 - - def _write_cahd_for_grain(self, grain): - self._file.write(b"cahd") - _write_uint(self._file, self._cahd_size_for_grain(grain), 4) + data += _encode_uint(grain.unit_offsets[i], 4) - _write_uint(self._file, int(grain.format), 4) - _write_uint(self._file, int(grain.channels), 2) - _write_uint(self._file, int(grain.samples), 4) - _write_uint(self._file, int(grain.priming), 4) - _write_uint(self._file, int(grain.remainder), 4) - _write_uint(self._file, int(grain.sample_rate), 4) + return data - def complete_write(self): - if self._file is None: - return + def _cahd_size_for_grain(self, grain: CODEDAUDIOGRAIN) -> int: + return 30 - if seekable(self._file) and self._count_pos != -1: - curpos = self._file.tell() - self._file.seek(self._count_pos) - _write_sint(self._file, self._write_count, 8) - self._file.seek(curpos) + def _encode_cahd_for_grain(self, grain: CODEDAUDIOGRAIN) -> bytes: + return (b"cahd" + + _encode_uint(self._cahd_size_for_grain(grain), 4) + - self._file = None - self._count_pos = -1 + _encode_uint(int(grain.format), 4) + + _encode_uint(int(grain.channels), 2) + + _encode_uint(int(grain.samples), 4) + + _encode_uint(int(grain.priming), 4) + + _encode_uint(int(grain.remainder), 4) + + _encode_uint(int(grain.sample_rate), 4)) - def add_tag(self, key, value): + def add_tag(self, key: str, value: str): """Add a tag to the segment""" - if self._file is not None: + if self._active_dump: raise GSFEncodeAddToActiveDump("Cannot add a tag to a segment which is part of an active export") self._tags.append(GSFEncoderTag(key, value)) - def add_grain(self, grain): + def add_grain(self, grain: GRAIN): """Add a grain to the segment, which should be a Grain object""" - if self._file is not None: - self._write_grain(grain) - else: - self._grains.append(grain) + self._grains.append(grain) - def add_grains(self, grains): + def add_grains(self, grains: Iterable[GRAIN]): """Add several grains to the segment, the parameter should be an iterable of grain objects""" for grain in grains: diff --git a/mediagrains/hypothesis/py.typed b/mediagrains/hypothesis/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains/hypothesis/strategies.py b/mediagrains/hypothesis/strategies.py index e1c0de2..2283726 100644 --- a/mediagrains/hypothesis/strategies.py +++ b/mediagrains/hypothesis/strategies.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2018 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,9 +18,6 @@ generate mediagrains for hypothesis based testing """ -from __future__ import print_function -from __future__ import absolute_import - from mediatimestamp.hypothesis.strategies import immutabletimestamps as timestamps from hypothesis.strategies import ( integers, diff --git a/mediagrains_py36/numpy/__init__.py b/mediagrains/numpy/__init__.py similarity index 94% rename from mediagrains_py36/numpy/__init__.py rename to mediagrains/numpy/__init__.py index 662a315..1834e90 100644 --- a/mediagrains_py36/numpy/__init__.py +++ b/mediagrains/numpy/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +18,6 @@ """ from .videograin import VIDEOGRAIN, VideoGrain -from . import convert +from . import convert # noqa: F401 __all__ = ['VideoGrain', 'VIDEOGRAIN'] diff --git a/mediagrains_py36/numpy/convert.py b/mediagrains/numpy/convert.py similarity index 84% rename from mediagrains_py36/numpy/convert.py rename to mediagrains/numpy/convert.py index d0dea23..b9d97d6 100644 --- a/mediagrains_py36/numpy/convert.py +++ b/mediagrains/numpy/convert.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,15 +17,12 @@ Library for converting video grain formats represented as numpy arrays. """ -from mediagrains.cogenums import CogFrameFormat, CogFrameLayout, COG_FRAME_FORMAT_ACTIVE_BITS, COG_PLANAR_FORMAT, PlanarChromaFormat -from typing import Callable, List -from uuid import uuid5, UUID +from mediagrains.cogenums import CogFrameFormat, COG_FRAME_FORMAT_ACTIVE_BITS, COG_PLANAR_FORMAT, PlanarChromaFormat +from typing import List, Tuple, Sequence import numpy as np import numpy.random as npr -from pdb import set_trace - -from .videograin import VideoGrain, VIDEOGRAIN +from .videograin import VIDEOGRAIN def distinct_pairs_from(vals): @@ -36,7 +31,9 @@ def distinct_pairs_from(vals): yield (vals[i], vals[j]) -def compose(first: Callable[[VIDEOGRAIN, VIDEOGRAIN], None], intermediate: CogFrameFormat, second: Callable[[VIDEOGRAIN, VIDEOGRAIN], None]) -> Callable[[VIDEOGRAIN, VIDEOGRAIN], None]: +def compose(first: VIDEOGRAIN.ConversionFunc, + intermediate: CogFrameFormat, + second: VIDEOGRAIN.ConversionFunc) -> VIDEOGRAIN.ConversionFunc: """Compose two conversion functions together""" def _inner(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): grain_intermediate = grain_in._similar_grain(intermediate) @@ -49,39 +46,39 @@ def _inner(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): # Some simple conversions can be acheived by just copying the data from one grain to the other with no # clever work at all. All the cleverness is already present in the code that creates the component array views # in the mediagrains -def _simple_copy_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.Y[:,:] = grain_in.component_data.Y - grain_out.component_data.U[:,:] = grain_in.component_data.U - grain_out.component_data.V[:,:] = grain_in.component_data.V +def _simple_copy_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.Y[:, :] = grain_in.component_data.Y + grain_out.component_data.U[:, :] = grain_in.component_data.U + grain_out.component_data.V[:, :] = grain_in.component_data.V -def _simple_copy_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.R[:,:] = grain_in.component_data.R - grain_out.component_data.G[:,:] = grain_in.component_data.G - grain_out.component_data.B[:,:] = grain_in.component_data.B +def _simple_copy_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.R[:, :] = grain_in.component_data.R + grain_out.component_data.G[:, :] = grain_in.component_data.G + grain_out.component_data.B[:, :] = grain_in.component_data.B def _int_array_mean(a: np.ndarray, b: np.ndarray) -> np.ndarray: """This takes the mean of two arrays of integers without risking overflowing intermediate values.""" - return (a//2 + b//2) + ((a&0x1) | (b&0x1)) + return (a//2 + b//2) + ((a & 0x1) | (b & 0x1)) # Some conversions between YUV colour subsampling systems require a simple mean -def _simple_mean_convert_yuv444__yuv422(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.Y[:,:] = grain_in.component_data.Y - grain_out.component_data.U[:,:] = _int_array_mean(grain_in.component_data.U[0::2, :], grain_in.component_data.U[1::2, :]) - grain_out.component_data.V[:,:] = _int_array_mean(grain_in.component_data.V[0::2, :], grain_in.component_data.V[1::2, :]) +def _simple_mean_convert_yuv444__yuv422(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.Y[:, :] = grain_in.component_data.Y + grain_out.component_data.U[:, :] = _int_array_mean(grain_in.component_data.U[0::2, :], grain_in.component_data.U[1::2, :]) + grain_out.component_data.V[:, :] = _int_array_mean(grain_in.component_data.V[0::2, :], grain_in.component_data.V[1::2, :]) -def _simple_mean_convert_yuv422__yuv420(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.Y[:,:] = grain_in.component_data.Y - grain_out.component_data.U[:,:] = _int_array_mean(grain_in.component_data.U[:, 0::2], grain_in.component_data.U[:, 1::2]) - grain_out.component_data.V[:,:] = _int_array_mean(grain_in.component_data.V[:, 0::2], grain_in.component_data.V[:, 1::2]) +def _simple_mean_convert_yuv422__yuv420(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.Y[:, :] = grain_in.component_data.Y + grain_out.component_data.U[:, :] = _int_array_mean(grain_in.component_data.U[:, 0::2], grain_in.component_data.U[:, 1::2]) + grain_out.component_data.V[:, :] = _int_array_mean(grain_in.component_data.V[:, 0::2], grain_in.component_data.V[:, 1::2]) # Other conversions require duplicating samples -def _simple_duplicate_convert_yuv422__yuv444(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.Y[:,:] = grain_in.component_data.Y +def _simple_duplicate_convert_yuv422__yuv444(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.Y[:, :] = grain_in.component_data.Y grain_out.component_data.U[0::2, :] = grain_in.component_data.U grain_out.component_data.U[1::2, :] = grain_in.component_data.U @@ -89,8 +86,8 @@ def _simple_duplicate_convert_yuv422__yuv444(grain_in: VIDEOGRAIN, grain_out: VI grain_out.component_data.V[1::2, :] = grain_in.component_data.V -def _simple_duplicate_convert_yuv420__yuv422(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): - grain_out.component_data.Y[:,:] = grain_in.component_data.Y +def _simple_duplicate_convert_yuv420__yuv422(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: + grain_out.component_data.Y[:, :] = grain_in.component_data.Y grain_out.component_data.U[:, 0::2] = grain_in.component_data.U grain_out.component_data.U[:, 1::2] = grain_in.component_data.U @@ -100,16 +97,18 @@ def _simple_duplicate_convert_yuv420__yuv422(grain_in: VIDEOGRAIN, grain_out: VI # Bit depth conversions def _unbiased_right_shift(a: np.ndarray, n: int) -> np.ndarray: - return (a >> n) + ((a >> (n - 1))&0x1) + return (a >> n) + ((a >> (n - 1)) & 0x1) -def _bitdepth_down_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): + +def _bitdepth_down_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bitshift = COG_FRAME_FORMAT_ACTIVE_BITS(grain_in.format) - COG_FRAME_FORMAT_ACTIVE_BITS(grain_out.format) grain_out.component_data[0][:] = _unbiased_right_shift(grain_in.component_data[0][:], bitshift) grain_out.component_data[1][:] = _unbiased_right_shift(grain_in.component_data[1][:], bitshift) grain_out.component_data[2][:] = _unbiased_right_shift(grain_in.component_data[2][:], bitshift) -def _bitdepth_down_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): + +def _bitdepth_down_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bitshift = COG_FRAME_FORMAT_ACTIVE_BITS(grain_in.format) - COG_FRAME_FORMAT_ACTIVE_BITS(grain_out.format) grain_out.component_data.R[:] = _unbiased_right_shift(grain_in.component_data.R[:], bitshift) @@ -121,7 +120,8 @@ def _noisy_left_shift(a: np.ndarray, n: int) -> np.ndarray: rando = ((npr.random_sample(a.shape) * (1 << n)).astype(a.dtype)) & ((1 << n) - 1) return (a << n) + rando -def _bitdepth_up_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): + +def _bitdepth_up_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bitshift = COG_FRAME_FORMAT_ACTIVE_BITS(grain_out.format) - COG_FRAME_FORMAT_ACTIVE_BITS(grain_in.format) dt = grain_out.component_data[0].dtype @@ -130,7 +130,8 @@ def _bitdepth_up_convert_yuv(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): grain_out.component_data[1][:] = _noisy_left_shift(grain_in.component_data[1][:].astype(dt), bitshift) grain_out.component_data[2][:] = _noisy_left_shift(grain_in.component_data[2][:].astype(dt), bitshift) -def _bitdepth_up_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): + +def _bitdepth_up_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bitshift = COG_FRAME_FORMAT_ACTIVE_BITS(grain_out.format) - COG_FRAME_FORMAT_ACTIVE_BITS(grain_in.format) dt = grain_out.component_data[0].dtype @@ -141,29 +142,29 @@ def _bitdepth_up_convert_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): # Colourspace conversions (based on rec.709) -def _convert_rgb_to_yuv444(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): +def _convert_rgb_to_yuv444(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bd = COG_FRAME_FORMAT_ACTIVE_BITS(grain_out.format) (R, G, B) = (grain_in.component_data.R, - grain_in.component_data.G, - grain_in.component_data.B) + grain_in.component_data.G, + grain_in.component_data.B) - np.clip((R*0.2126 + G*0.7152 + B*0.0722), 0, 1 << bd, out=grain_out.component_data.Y, casting="unsafe") + np.clip((R*0.2126 + G*0.7152 + B*0.0722), 0, 1 << bd, out=grain_out.component_data.Y, casting="unsafe") np.clip((R*-0.114572 - G*0.385428 + B*0.5 + (1 << (bd - 1))), 0, 1 << bd, out=grain_out.component_data.U, casting="unsafe") - np.clip((R*0.5 - G*0.454153 - B*0.045847 + (1 << (bd - 1))), 0, 1 << bd, out=grain_out.component_data.V, casting="unsafe") + np.clip((R*0.5 - G*0.454153 - B*0.045847 + (1 << (bd - 1))), 0, 1 << bd, out=grain_out.component_data.V, casting="unsafe") -def _convert_yuv444_to_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): +def _convert_yuv444_to_rgb(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: bd = COG_FRAME_FORMAT_ACTIVE_BITS(grain_in.format) (Y, U, V) = (grain_in.component_data.Y.astype(np.dtype(np.double)), - grain_in.component_data.U.astype(np.dtype(np.double)) - (1 << (bd - 1)), - grain_in.component_data.V.astype(np.dtype(np.double)) - (1 << (bd - 1))) + grain_in.component_data.U.astype(np.dtype(np.double)) - (1 << (bd - 1)), + grain_in.component_data.V.astype(np.dtype(np.double)) - (1 << (bd - 1))) np.clip((Y + V*1.5748), 0, 1 << bd, out=grain_out.component_data.R, casting="unsafe") np.clip((Y - U*0.187324 - V*0.468124), 0, 1 << bd, out=grain_out.component_data.G, casting="unsafe") np.clip((Y + U*1.8556), 0, 1 << bd, out=grain_out.component_data.B, casting="unsafe") -def _convert_v210_to_yuv422_10bit(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): +def _convert_v210_to_yuv422_10bit(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: # This is a v210 -> planar descramble. It's not super fast, but it should be correct # # Input data is array of 32-bit words, arranged as a 1d array in repeating blocks of 4 like: @@ -179,9 +180,9 @@ def _convert_v210_to_yuv422_10bit(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): # second = [Y0] [U1] [Y3] [V2] ... # third = [V0] [Y2] [U2] [Y5] ... - first = (grain_in.data & 0x3FF).astype(np.dtype(np.uint16)) + first = (grain_in.data & 0x3FF).astype(np.dtype(np.uint16)) second = ((grain_in.data >> 10) & 0x3FF).astype(np.dtype(np.uint16)) - third = ((grain_in.data >> 20) & 0x3FF).astype(np.dtype(np.uint16)) + third = ((grain_in.data >> 20) & 0x3FF).astype(np.dtype(np.uint16)) # These arrays are still linear 1d arrays so we reinterpret them as 2d arrays, remembering that v210 has an alignment of 48 pixels horizontally first.shape = (grain_in.height, 32*((grain_in.width + 47)//48)) @@ -222,7 +223,7 @@ def _convert_v210_to_yuv422_10bit(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): grain_out.component_data.V[2::3, :] = second[3::4, :][0:(grain_in.width//2 + 0)//3, :] -def _convert_yuv422_10bit_to_v210(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): +def _convert_yuv422_10bit_to_v210(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN) -> None: # This won't be fast, but it should work. # Take every third entry in each component and arrange them @@ -247,21 +248,22 @@ def _convert_yuv422_10bit_to_v210(grain_in: VIDEOGRAIN, grain_out: VIDEOGRAIN): # These methods automate the process of registering simple copy conversions -def _register_simple_copy_conversions_for_formats_yuv(fmts: List[CogFrameFormat]): +def _register_simple_copy_conversions_for_formats_yuv(fmts: Sequence[CogFrameFormat]): for i in range(0, len(fmts)): for j in range(i+1, len(fmts)): VIDEOGRAIN.grain_conversion(fmts[i], fmts[j])(_simple_copy_convert_yuv) VIDEOGRAIN.grain_conversion(fmts[j], fmts[i])(_simple_copy_convert_yuv) -def _register_simple_copy_conversions_for_formats_rgb(fmts: List[CogFrameFormat]): + +def _register_simple_copy_conversions_for_formats_rgb(fmts: Sequence[CogFrameFormat]): for i in range(0, len(fmts)): for j in range(i+1, len(fmts)): VIDEOGRAIN.grain_conversion(fmts[i], fmts[j])(_simple_copy_convert_rgb) VIDEOGRAIN.grain_conversion(fmts[j], fmts[i])(_simple_copy_convert_rgb) -def _equivalent_formats(fmt: CogFrameFormat) -> List[CogFrameFormat]: - equiv_categories = [ +def _equivalent_formats(fmt: CogFrameFormat) -> Tuple[CogFrameFormat, ...]: + equiv_categories: List[Tuple[CogFrameFormat, ...]] = [ (CogFrameFormat.U8_422, CogFrameFormat.UYVY, CogFrameFormat.YUYV), (CogFrameFormat.S16_422, CogFrameFormat.v216), (CogFrameFormat.RGB, CogFrameFormat.U8_444_RGB, CogFrameFormat.RGBx, CogFrameFormat.xRGB, CogFrameFormat.BGRx, CogFrameFormat.xBGR)] @@ -283,8 +285,10 @@ def _equivalent_formats(fmt: CogFrameFormat) -> List[CogFrameFormat]: VIDEOGRAIN.grain_conversion(fmt, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd))(_simple_mean_convert_yuv422__yuv420) VIDEOGRAIN.grain_conversion(COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd), fmt)(_simple_duplicate_convert_yuv420__yuv422) VIDEOGRAIN.grain_conversion(fmt, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_444, bd))(_simple_duplicate_convert_yuv422__yuv444) - VIDEOGRAIN.grain_conversion(COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_444, bd), COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd))(compose(_simple_mean_convert_yuv444__yuv422, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_422, bd), _simple_mean_convert_yuv422__yuv420)) - VIDEOGRAIN.grain_conversion(COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd), COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_444, bd))(compose(_simple_duplicate_convert_yuv420__yuv422, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_422, bd), _simple_duplicate_convert_yuv422__yuv444)) + VIDEOGRAIN.grain_conversion(COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_444, bd), COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd))( + compose(_simple_mean_convert_yuv444__yuv422, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_422, bd), _simple_mean_convert_yuv422__yuv420)) + VIDEOGRAIN.grain_conversion(COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_420, bd), COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_444, bd))( + compose(_simple_duplicate_convert_yuv420__yuv422, COG_PLANAR_FORMAT(PlanarChromaFormat.YUV_422, bd), _simple_duplicate_convert_yuv422__yuv444)) # Bit depth conversions @@ -362,4 +366,4 @@ def _equivalent_formats(fmt: CogFrameFormat) -> List[CogFrameFormat]: for fmt in _equivalent_formats(COG_PLANAR_FORMAT(ss, d)): if fmt != CogFrameFormat.S16_422_10BIT: VIDEOGRAIN.grain_conversion_two_step(CogFrameFormat.v210, CogFrameFormat.S16_422_10BIT, fmt) - VIDEOGRAIN.grain_conversion_two_step(fmt, CogFrameFormat.S16_422_10BIT, CogFrameFormat.v210) \ No newline at end of file + VIDEOGRAIN.grain_conversion_two_step(fmt, CogFrameFormat.S16_422_10BIT, CogFrameFormat.v210) diff --git a/mediagrains/numpy/py.typed b/mediagrains/numpy/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains_py36/numpy/videograin.py b/mediagrains/numpy/videograin.py similarity index 66% rename from mediagrains_py36/numpy/videograin.py rename to mediagrains/numpy/videograin.py index 5154ce3..56cf0f4 100644 --- a/mediagrains_py36/numpy/videograin.py +++ b/mediagrains/numpy/videograin.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,10 +17,9 @@ Library for handling mediagrains in numpy arrays """ -from mediagrains.cogenums import ( +from ..cogenums import ( CogFrameFormat, - COG_FRAME_IS_PACKED, - COG_FRAME_IS_COMPRESSED, + CogFrameLayout, COG_FRAME_IS_PLANAR, COG_FRAME_FORMAT_BYTES_PER_VALUE, COG_FRAME_IS_PLANAR_RGB) @@ -34,9 +31,15 @@ import numpy as np from numpy.lib.stride_tricks import as_strided -from typing import Callable +from typing import Callable, Dict, Tuple, Optional, Awaitable, cast, overload +from ..typing import VideoGrainMetadataDict, GrainDataType, GrainDataParameterType + +from inspect import isawaitable from enum import Enum, auto +from uuid import UUID +from fractions import Fraction +from mediatimestamp.immutable import Timestamp __all__ = ['VideoGrain', 'VIDEOGRAIN'] @@ -82,9 +85,9 @@ class ComponentOrder (Enum): YUV = auto() RGB = auto() BGR = auto() - X = auto() + X = auto() - def __init__(self, data: list, arrangement: ComponentOrder=ComponentOrder.X): + def __init__(self, data: list, arrangement: ComponentOrder = ComponentOrder.X): super().__init__(data) if arrangement == ComponentDataList.ComponentOrder.YUV: self.Y = self[0] @@ -136,7 +139,14 @@ def _component_arrays_for_interleaved_422(data0: np.ndarray, data1: np.ndarray, strides=(stride, itemsize*4)).transpose()] -def _component_arrays_for_interleaved_444_take_three(data0: np.ndarray, data1: np.ndarray, data2: np.ndarray, width: int, height: int, stride: int, itemsize: int, num_components: int = 3): +def _component_arrays_for_interleaved_444_take_three(data0: np.ndarray, + data1: np.ndarray, + data2: np.ndarray, + width: int, + height: int, + stride: int, + itemsize: int, + num_components: int = 3): return [ as_strided(data0, shape=(height, width), @@ -173,20 +183,40 @@ def _component_arrays_for_data_and_type(data: np.ndarray, fmt: CogFrameFormat, c return _component_arrays_for_interleaved_422(data, data[1:], data[3:], components[0].width, components[0].height, components[0].stride, data.itemsize) elif fmt == CogFrameFormat.RGB: # 8 bit 4:4:4 three components interleaved in RGB order - return _component_arrays_for_interleaved_444_take_three(data, data[1:], data[2:], components[0].width, components[0].height, components[0].stride, data.itemsize) + return _component_arrays_for_interleaved_444_take_three(data, + data[1:], + data[2:], + components[0].width, + components[0].height, + components[0].stride, + data.itemsize) elif fmt in [CogFrameFormat.RGBx, CogFrameFormat.RGBA, CogFrameFormat.BGRx, CogFrameFormat.BGRx]: # 8 bit 4:4:4:4 four components interleave dropping the fourth component - return _component_arrays_for_interleaved_444_take_three(data, data[1:], data[2:], components[0].width, components[0].height, components[0].stride, data.itemsize, num_components=4) + return _component_arrays_for_interleaved_444_take_three(data, + data[1:], + data[2:], + components[0].width, + components[0].height, + components[0].stride, + data.itemsize, + num_components=4) elif fmt in [CogFrameFormat.ARGB, CogFrameFormat.xRGB, CogFrameFormat.ABGR, CogFrameFormat.xBGR, CogFrameFormat.AYUV]: # 8 bit 4:4:4:4 four components interleave dropping the first component - return _component_arrays_for_interleaved_444_take_three(data[1:], data[2:], data[3:], components[0].width, components[0].height, components[0].stride, data.itemsize, num_components=4) + return _component_arrays_for_interleaved_444_take_three(data[1:], + data[2:], + data[3:], + components[0].width, + components[0].height, + components[0].stride, + data.itemsize, + num_components=4) elif fmt == CogFrameFormat.v210: # v210 is barely supported. Convert it to something else to actually use it! # This method returns an empty list because component access isn't supported, but @@ -197,37 +227,75 @@ def _component_arrays_for_data_and_type(data: np.ndarray, fmt: CogFrameFormat, c class VIDEOGRAIN (bytesgrain.VIDEOGRAIN): - _grain_conversions = {} + ConversionFunc = Callable[["VIDEOGRAIN", "VIDEOGRAIN"], None] - def __init__(self, meta, data): + _grain_conversions: Dict[Tuple[CogFrameFormat, CogFrameFormat], ConversionFunc] = {} + + def __init__(self, meta: VideoGrainMetadataDict, data: GrainDataParameterType): super().__init__(meta, data) - self._data = np.frombuffer(self._data, dtype=_dtype_from_cogframeformat(self.format)) - self.component_data = ComponentDataList( - _component_arrays_for_data_and_type(self._data, self.format, self.components), - arrangement=_component_arrangement_from_format(self.format)) + self._data: np.ndarray + self._data_fetcher_coroutine: Optional[Awaitable[GrainDataType]] + self.component_data: ComponentDataList + + if self._data is not None: + self._data = np.frombuffer(self._data, dtype=_dtype_from_cogframeformat(self.format)) + self.component_data = ComponentDataList( + _component_arrays_for_data_and_type(self._data, self.format, self.components), + arrangement=_component_arrangement_from_format(self.format)) + else: + self.component_data = ComponentDataList([]) + + @property + def data(self) -> np.ndarray: + return self._data + + @data.setter + def data(self, value: GrainDataParameterType): + if isawaitable(value): + self._data_fetcher_coroutine = cast(Awaitable[GrainDataType], value) + self._data = None + self.component_data = ComponentDataList([]) + else: + self._data_fetcher_coroutine = None + self._data = np.frombuffer(cast(GrainDataType, value), dtype=_dtype_from_cogframeformat(self.format)) + self.component_data = ComponentDataList( + _component_arrays_for_data_and_type(self._data, self.format, self.components), + arrangement=_component_arrangement_from_format(self.format)) - def __array__(self): + def __array__(self) -> np.ndarray: return np.array(self.data) - def __bytes__(self): - return bytes(self.data) + def __bytes__(self) -> bytes: + return bytes(self._data) - def __copy__(self): + def __copy__(self) -> "VIDEOGRAIN": return VideoGrain(copy(self.meta), self.data) - def __deepcopy__(self, memo): - return VideoGrain(deepcopy(self.meta), self.data.copy()) + def __deepcopy__(self, memo) -> "VIDEOGRAIN": + return VideoGrain(deepcopy(self.meta), self._data.copy()) - def __repr__(self): + def __repr__(self) -> str: if self.data is None: return "{}({!r})".format(self._factory, self.meta) else: return "{}({!r},< numpy data of length {} >)".format(self._factory, self.meta, len(self.data)) + async def __await__(self): + if self._data is None and self._data_fetcher_coroutine is not None: + self.data = await self._data_fetcher_coroutine + return self.data + + async def __aenter__(self) -> "VIDEOGRAIN": + await self + return self + + async def __aexit__(self, *args, **kwargs): + pass + @classmethod - def grain_conversion(cls, fmt_in: CogFrameFormat, fmt_out: CogFrameFormat): + def grain_conversion(cls, fmt_in: CogFrameFormat, fmt_out: CogFrameFormat) -> Callable[["VIDEOGRAIN.ConversionFunc"], "VIDEOGRAIN.ConversionFunc"]: """Decorator to apply to all grain conversion functions""" - def _inner(f: Callable[[cls, cls], None]) -> None: + def _inner(f: "VIDEOGRAIN.ConversionFunc") -> "VIDEOGRAIN.ConversionFunc": cls._grain_conversions[(fmt_in, fmt_out)] = f return f return _inner @@ -242,7 +310,7 @@ def _inner(grain_in: "VIDEOGRAIN", grain_out: "VIDEOGRAIN"): cls.grain_conversion(fmt_in, fmt_out)(_inner) @classmethod - def _get_grain_conversion_function(cls, fmt_in: CogFrameFormat, fmt_out: CogFrameFormat) -> Callable[["VIDEOGRAIN", "VIDEOGRAIN"], None]: + def _get_grain_conversion_function(cls, fmt_in: CogFrameFormat, fmt_out: CogFrameFormat) -> "VIDEOGRAIN.ConversionFunc": """Return the registered grain conversion function for a specified type conversion, or raise NotImplementedError""" if (fmt_in, fmt_out) in cls._grain_conversions: return cls._grain_conversions[(fmt_in, fmt_out)] @@ -292,7 +360,36 @@ def asformat(self, fmt: CogFrameFormat) -> "VIDEOGRAIN": return self.convert(fmt) -def VideoGrain(*args, **kwargs) -> VIDEOGRAIN: +@overload +def VideoGrain(grain: bytesgrain.VIDEOGRAIN) -> VIDEOGRAIN: ... + + +@overload +def VideoGrain(src_id_or_meta: VideoGrainMetadataDict, + flow_id_or_data: GrainDataParameterType = None) -> VIDEOGRAIN: ... + + +@overload +def VideoGrain(src_id_or_meta: Optional[UUID] = None, + flow_id_or_data: Optional[UUID] = None, + origin_timestamp: Optional[Timestamp] = None, + creation_timestamp: Optional[Timestamp] = None, + sync_timestamp: Optional[Timestamp] = None, + rate: Fraction = Fraction(25, 1), + duration: Fraction = Fraction(1, 25), + cog_frame_format: CogFrameFormat = CogFrameFormat.UNKNOWN, + width: int = 1920, + height: int = 1080, + cog_frame_layout: CogFrameLayout = CogFrameLayout.UNKNOWN, + src_id: Optional[UUID] = None, + source_id: Optional[UUID] = None, + format: Optional[CogFrameFormat] = None, + layout: Optional[CogFrameLayout] = None, + flow_id: Optional[UUID] = None, + data: GrainDataParameterType = None) -> VIDEOGRAIN: ... + + +def VideoGrain(*args, **kwargs): """If the first argument is a mediagrains.VIDEOGRAIN then return a mediagrains.numpy.VIDEOGRAIN representing the same data. Otherwise takes the same parameters as mediagrains.VideoGrain and returns the same grain converted into a mediagrains.numpy.VIDEOGRAIN @@ -302,4 +399,7 @@ def VideoGrain(*args, **kwargs) -> VIDEOGRAIN: else: rawgrain = bytesgrain_constructors.VideoGrain(*args, **kwargs) - return VIDEOGRAIN(rawgrain.meta, rawgrain.data) + if rawgrain.data is not None: + return VIDEOGRAIN(rawgrain.meta, rawgrain.data) + else: + return VIDEOGRAIN(rawgrain.meta, rawgrain._data_fetcher_coroutine) diff --git a/mediagrains/py.typed b/mediagrains/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains/testsignalgenerator.py b/mediagrains/testsignalgenerator.py index fc423a5..b25bab6 100644 --- a/mediagrains/testsignalgenerator.py +++ b/mediagrains/testsignalgenerator.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -19,9 +18,6 @@ grains. """ -from __future__ import print_function -from __future__ import absolute_import - from fractions import Fraction from mediatimestamp.immutable import TimeOffset from copy import deepcopy @@ -124,7 +120,6 @@ def LumaSteps(src_id, flow_id, width, height, vg.sync_timestamp = vg.origin_timestamp - def ColourBars(src_id, flow_id, width, height, intensity=0.75, rate=Fraction(25, 1), @@ -175,10 +170,12 @@ def ColourBars(src_id, flow_id, width, height, lines[c][2*x + 0] = values[pos][c] & 0xFF lines[c][2*x + 1] = (values[pos][c] >> 8) & 0xFF - for c in range(0, 3): for y in range(0, vg.components[c].height): - vg.data[vg.components[c].offset + y*vg.components[c].stride:vg.components[c].offset + y*vg.components[c].stride + vg.components[c].width*_bpp] = lines[c] + vg.data[vg.components[c].offset + + y*vg.components[c].stride:vg.components[c].offset + + y*vg.components[c].stride + + vg.components[c].width*_bpp] = lines[c] origin_timestamp = vg.origin_timestamp count = 0 @@ -209,7 +206,9 @@ def MovingBarOverlay(grain_gen, height=100, speed=1.0): _bpp = pixel_ranges[grain.format][0] - bar = [bytearray(grain.components[0].width*_bpp * height), bytearray(grain.components[1].width*_bpp * height // v_subs), bytearray(grain.components[2].width*_bpp * height // v_subs)] + bar = [bytearray(grain.components[0].width*_bpp * height), + bytearray(grain.components[1].width*_bpp * height // v_subs), + bytearray(grain.components[2].width*_bpp * height // v_subs)] for y in range(0, height): for x in range(0, grain.components[0].width): bar[0][y*grain.components[0].width * _bpp + _bpp*x + 0] = pixel_ranges[grain.format][1][0] & 0xFF @@ -229,19 +228,18 @@ def MovingBarOverlay(grain_gen, height=100, speed=1.0): for y in range(0, height): grain.data[ grain.components[0].offset + ((fnum + y) % grain.components[0].height)*grain.components[0].stride: - grain.components[0].offset + ((fnum + y) % grain.components[0].height)*grain.components[0].stride + grain.components[0].width*_bpp ] = ( + grain.components[0].offset + ((fnum + y) % grain.components[0].height)*grain.components[0].stride + grain.components[0].width*_bpp] = ( bar[0][y*grain.components[0].width * _bpp: (y+1)*grain.components[0].width * _bpp]) for y in range(0, height // v_subs): grain.data[ grain.components[1].offset + ((fnum//v_subs + y) % grain.components[1].height)*grain.components[1].stride: - grain.components[1].offset + ((fnum//v_subs + y) % grain.components[1].height)*grain.components[1].stride + grain.components[1].width*_bpp ] = ( + grain.components[1].offset + ((fnum//v_subs + y) % grain.components[1].height)*grain.components[1].stride + grain.components[1].width*_bpp] = ( bar[1][y*grain.components[1].width * _bpp: (y+1)*grain.components[1].width * _bpp]) grain.data[ grain.components[2].offset + ((fnum//v_subs + y) % grain.components[2].height)*grain.components[2].stride: - grain.components[2].offset + ((fnum//v_subs + y) % grain.components[2].height)*grain.components[2].stride + grain.components[2].width*_bpp ] = ( + grain.components[2].offset + ((fnum//v_subs + y) % grain.components[2].height)*grain.components[2].stride + grain.components[2].width*_bpp] = ( bar[2][y*grain.components[2].width * _bpp: (y+1)*grain.components[2].width * _bpp]) - yield grain diff --git a/mediagrains/numpy.py b/mediagrains/tools/__init__.py similarity index 62% rename from mediagrains/numpy.py rename to mediagrains/tools/__init__.py index 725983c..d8bc995 100644 --- a/mediagrains/numpy.py +++ b/mediagrains/tools/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,17 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - -"""\ -Numpy compatible layer for mediagrains, but only available in python 3.6+ -""" - -from sys import version_info -if version_info[0] > 3 or (version_info[0] == 3 and version_info[1] >= 6): - from mediagrains_py36.numpy import VideoGrain, VIDEOGRAIN # noqa: F401 +from .wrap_in_gsf import wrap_video_in_gsf, wrap_audio_in_gsf +from .extract_from_gsf import extract_gsf_essence, gsf_probe - __all__ = ['VideoGrain', 'VIDEOGRAIN'] -else: - __all__ = [] +__all__ = ["wrap_video_in_gsf", "wrap_audio_in_gsf", "extract_gsf_essence", "gsf_probe"] diff --git a/mediagrains/tools/_file_or_pipe.py b/mediagrains/tools/_file_or_pipe.py new file mode 100644 index 0000000..6afcfa0 --- /dev/null +++ b/mediagrains/tools/_file_or_pipe.py @@ -0,0 +1,36 @@ +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utility function to open a file or a pipe""" +import typing + +import sys +from contextlib import contextmanager + + +@contextmanager +def file_or_pipe(file_or_pipe: str, mode: str) -> typing.Iterator[typing.IO[bytes]]: + """Context manager to open a file or stdin/stdout for binary operations + + :param file_or_pipe: Name of file to open, or "-" to indicate a pipe + :param mode: Mode in which to open the given file or pipe - used directly for files and to detect direction of + of pipes. Must be one of "rb" or "wb" + """ + if file_or_pipe == "-": + if "w" in mode: + yield sys.stdout.buffer + else: + yield sys.stdin.buffer + else: + with open(file_or_pipe, mode) as fp: + yield fp diff --git a/mediagrains/tools/extract_from_gsf.py b/mediagrains/tools/extract_from_gsf.py new file mode 100644 index 0000000..5d2243c --- /dev/null +++ b/mediagrains/tools/extract_from_gsf.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Given GSF file, dump out the raw essence""" + +import argparse +import sys +import mediajson + +from ..gsf import GSFDecoder +from ._file_or_pipe import file_or_pipe + + +def extract_gsf_essence(): + """Provide a utility to extract raw essence from a GSF file""" + parser = argparse.ArgumentParser( + description="A utility to dump the essence data out of a GSF file" + ) + + parser.add_argument("input_file", help="Input file (stdin is not supported because it isn't seekable)", type=str) + parser.add_argument("output_file", help="Output GSF file path. Specify - for stdout", type=str) + + parser.add_argument("--only-id", help="Only include Grains with this GSF local ID. May be specified more than once", + type=int, action="append", default=None) + + args = parser.parse_args() + + with open(args.input_file, "rb") as input_data, file_or_pipe(args.output_file, "wb") as output_data: + decoder = GSFDecoder(file_data=input_data) + decoder.decode_file_headers() + + for grain, local_id in decoder.grains(local_ids=args.only_id): + print("Got grain with local_id {} at {}".format(local_id, grain.origin_timestamp.to_sec_nsec()), + file=sys.stderr) + output_data.write(grain.data) + + +def gsf_probe(): + """Provide a utility to dump information about a GSF file""" + parser = argparse.ArgumentParser( + description="A utility to dump the metadata out of a GSF file" + ) + + parser.add_argument("input_file", help="Input file (stdin is not supported because it isn't seekable)", type=str) + + args = parser.parse_args() + + with open(args.input_file, "rb") as input_data: + decoder = GSFDecoder(file_data=input_data) + file_data = decoder.decode_file_headers() + + file_data["segments"] = {segment["local_id"]: segment for segment in file_data["segments"]} + file_data["created"] = str(file_data["created"]) # Work around mediajson's inability to serialize datetimes + + for grain, local_id in decoder.grains(load_lazily=True): + this_segment = file_data["segments"][local_id] + + try: + this_segment["timerange"] = \ + this_segment["timerange"].extend_to_encompass_timerange(grain.origin_timerange()) + except KeyError: + this_segment["timerange"] = grain.origin_timerange() + this_segment["grain_data"] = grain.meta["grain"] + + print(mediajson.dumps(file_data, indent=True)) diff --git a/mediagrains/tools/py.typed b/mediagrains/tools/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains/tools/wrap_in_gsf.py b/mediagrains/tools/wrap_in_gsf.py new file mode 100644 index 0000000..af65467 --- /dev/null +++ b/mediagrains/tools/wrap_in_gsf.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Given a raw essence input, wrap it into a GSF file""" + +import uuid +import argparse +import sys + +from mediatimestamp.immutable import Timestamp + +from ..cogenums import CogFrameFormat, CogAudioFormat +from ..grain_constructors import VideoGrain, AudioGrain +from ..grain import GRAIN +from ..gsf import GSFEncoder +from ..utils import GrainWrapper +from ._file_or_pipe import file_or_pipe + + +def wrap_to_gsf( + input_file: str, + output_file: str, + template_grain: GRAIN): + """Wrap the supplied input in GSF and write it out to a given file-like object + + :param input_file: A file path (or "-" for stdin) to read the input media from, one frame/Grain at a time + :param output_file: A file path (or "-" for stdout) to write output GSF data to + :param template_grain: Base Grain to use as a template for the others + """ + with file_or_pipe(input_file, "rb") as input_data, file_or_pipe(output_file, "wb") as output_data: + wrapper = GrainWrapper(template_grain, input_data) + + # Write a GSF file with the grains read from the input + encoder = GSFEncoder(output_data) + segment = encoder.add_segment(id=wrapper.template_grain.flow_id) + encoder.start_dump() + + for grain in wrapper.grains(): + print("Got grain with TS {}".format(grain.origin_timestamp.to_sec_nsec()), file=sys.stderr) + segment.add_grains([grain]) + + encoder.end_dump() + + +def wrap_video_in_gsf(): + """Provide a utility to take a raw video input and turn it into a GSF file""" + parser = argparse.ArgumentParser( + description="A utility to take raw video essence and generate a GSF file", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("input_file", help="Input file. Specify - for stdin", type=str) + parser.add_argument("output_file", help="Output GSF file path. Specify - for stdout", type=str) + + parser.add_argument("--flow-id", help="UUID of GSF Flow - one will be generated if not set", + type=uuid.UUID, default=None) + parser.add_argument("--source-id", help="UUID of GSF Source - one will be generated if not given", + type=uuid.UUID, default=None) + parser.add_argument("--start-ts", help="Timestamp of start of media", type=Timestamp.from_str, + default=Timestamp(0, 0)) + + parser.add_argument("--size", help="Size of input video, in WidthxHeight form", default="1920x1080") + + parser.add_argument("--format", help="Frame format; one of the CogFrameFormat options", + type=lambda x: CogFrameFormat[x], default=CogFrameFormat.S16_422_10BIT.name) + + parser.add_argument("--rate", help="Frame rate of input video", type=int, default=25) + + args = parser.parse_args() + + # Parse width and height separately + width, height = [int(element.strip()) for element in args.size.split("x")] + + # Generate missing UUIDs + flow_id = args.flow_id if args.flow_id else uuid.uuid4() + source_id = args.source_id if args.source_id else uuid.uuid4() + + template_grain = VideoGrain( + flow_id=flow_id, source_id=source_id, origin_timestamp=args.start_ts, rate=args.rate, + width=width, height=height, cog_frame_format=args.format + ) + + wrap_to_gsf(input_file=args.input_file, output_file=args.output_file, template_grain=template_grain) + + +def wrap_audio_in_gsf(): + """Provide a utility to take a raw audio input and turn it into a GSF file""" + parser = argparse.ArgumentParser( + description="A utility to take raw audio samples and generate a GSF file", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("input_file", help="Input file. Specify - for stdin", type=str) + parser.add_argument("output_file", help="Output GSF file path. Specify - for stdout", type=str) + + parser.add_argument("--flow-id", help="UUID of GSF Flow - one will be generated if not set", + type=uuid.UUID, default=None) + parser.add_argument("--source-id", help="UUID of GSF Source - one will be generated if not given", + type=uuid.UUID, default=None) + parser.add_argument("--start-ts", help="Timestamp of start of media", + type=Timestamp.from_str, default=Timestamp(0, 0)) + + parser.add_argument("--channels", help="Number of channels present in input media", type=int, default=2) + parser.add_argument("--samples-per-grain", help="Number of samples to write to each Grain", type=int, default=1920) + + parser.add_argument("--format", help="Audio format; one of the CogAudioFormat options", + type=lambda x: CogAudioFormat[x], default=CogAudioFormat.S16_PLANES.name) + + parser.add_argument("--sample-rate", help="Sample rate of input audio", type=int, default=48000) + + args = parser.parse_args() + + # Generate missing UUIDs + flow_id = args.flow_id if args.flow_id else uuid.uuid4() + source_id = args.source_id if args.source_id else uuid.uuid4() + + template_grain = AudioGrain( + flow_id=flow_id, source_id=source_id, origin_timestamp=args.start_ts, rate=args.sample_rate, + channels=args.channels, samples=args.samples_per_grain, cog_audio_format=args.format + ) + + wrap_to_gsf(input_file=args.input_file, output_file=args.output_file, template_grain=template_grain) diff --git a/mediagrains/typing.py b/mediagrains/typing.py new file mode 100644 index 0000000..dfae0c3 --- /dev/null +++ b/mediagrains/typing.py @@ -0,0 +1,251 @@ +# +# Copyright 2018 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""\ +Types used for type checking other parts of the library +""" + +from .cogenums import CogFrameFormat, CogAudioFormat, CogFrameLayout + +from typing import Any, Union, SupportsBytes, Sequence, Mapping, List, Optional, Awaitable +from typing_extensions import TypedDict, Literal + +from decimal import Decimal +from numbers import Rational +from fractions import Fraction +from uuid import UUID +from mediatimestamp.immutable import TimeOffset, TimeRange, Timestamp + + +__all__ = ["RationalTypes", + "MediaJSONSerialisable", + "EventGrainDatumDict", + "GrainMetadataDict", + "EmptyGrainMetadataDict", + "AudioGrainMetadataDict", + "CodedAudioGrainMetadataDict", + "VideoGrainMetadataDict", + "EventGrainMetadataDict", + "FractionDict", + "GrainDataType", + "GrainDataParameterType"] + +# These are the types that can be freely converted into a Fraction +RationalTypes = Union[str, float, Decimal, Rational] + +# TODO: Move this into mediajson, and make it actually describe what is serialisable. +# At current due to weaknesses in mypy this is rather limited and only provides type safety for a limited depth of json strucure +# +# Hopefully at some point in the future proper recursive type definitions will be supported +# Until that time we simply assume none of our json structures are all that deep +_MediaJSONSerialisable_value = Union[str, int, UUID, TimeOffset, TimeRange, Fraction] +_MediaJSONSerialisable0 = Union[_MediaJSONSerialisable_value, Sequence[Any], Mapping[str, Any]] # This means that type checking stops at the fourth level +_MediaJSONSerialisable1 = Union[_MediaJSONSerialisable_value, Sequence[_MediaJSONSerialisable0], Mapping[str, _MediaJSONSerialisable0]] +_MediaJSONSerialisable2 = Union[_MediaJSONSerialisable_value, Sequence[_MediaJSONSerialisable1], Mapping[str, _MediaJSONSerialisable1]] +_MediaJSONSerialisable3 = Union[_MediaJSONSerialisable_value, Sequence[_MediaJSONSerialisable2], Mapping[str, _MediaJSONSerialisable2]] +_MediaJSONSerialisable4 = Union[_MediaJSONSerialisable_value, Sequence[_MediaJSONSerialisable3], Mapping[str, _MediaJSONSerialisable3]] +MediaJSONSerialisable = _MediaJSONSerialisable4 + + +# This is weird, but is currently how you specifiy a structured dict with optional entries +# This defines what is allowable in a dictionary representation of an EventGrain data element +class _EventGrainDatumDict_MANDATORY (TypedDict): + path: str + + +class EventGrainDatumDict (_EventGrainDatumDict_MANDATORY, total=False): + pre: MediaJSONSerialisable + post: MediaJSONSerialisable + + +# This type defines a dictionary that can be converted into a Fraction by mediaJSON + +class FractionDict (TypedDict): + numerator: int + denominator: int + + +class TimeLabel (TypedDict, total=False): + tag: str + count: int + rate: FractionDict + drop_frame: bool + + +# This is the type that defines what can go in a grain metadata dict. +class _GrainGrainMetadataDict_common_MANDATORY (TypedDict): + source_id: Union[str, UUID] + flow_id: Union[str, UUID] + origin_timestamp: Union[str, Timestamp] + sync_timestamp: Union[str, Timestamp] + creation_timestamp: Union[str, Timestamp] + rate: Union[RationalTypes, Fraction, FractionDict] + duration: Union[RationalTypes, Fraction, FractionDict] + + +class _GrainGrainMetadataDict_common (_GrainGrainMetadataDict_common_MANDATORY, total=False): + timelabels: List[TimeLabel] + + +class EmptyGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['empty'] + + +class _GrainGrainMetadataDict_cogaudio (TypedDict): + format: Union[int, CogAudioFormat] # noqa: E701 + samples: int + channels: int + sample_rate: int + + +class _GrainGrainMetadataDict_cogcodedaudio (_GrainGrainMetadataDict_cogaudio): + priming: int + remainder: int + + +class AudioGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['audio'] + cog_audio: _GrainGrainMetadataDict_cogaudio + + +class CodedAudioGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['coded_audio'] + cog_coded_audio: _GrainGrainMetadataDict_cogcodedaudio + + +class VideoGrainComponentDict(TypedDict): + stride: int + offset: int + width: int + height: int + length: int + + +class _GrainGrainMetadataDict_cogframe_MANDATORY (TypedDict): + format: Union[int, CogFrameFormat] # noqa: E701 + width: int + height: int + layout: Union[int, CogFrameLayout] + extension: int + components: List[VideoGrainComponentDict] + + +class _GrainGrainMetadataDict_cogframe (_GrainGrainMetadataDict_cogframe_MANDATORY, total=False): + source_aspect_ratio: Union[FractionDict, Fraction, RationalTypes] + pixel_aspect_ratio: Union[FractionDict, Fraction, RationalTypes] + + +class VideoGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['video'] + cog_frame: _GrainGrainMetadataDict_cogframe + + +class _GrainGrainMetadataDict_cogcodedframe_MANDATORY (TypedDict): + format: Union[int, CogFrameFormat] # noqa: E701 + origin_width: int + origin_height: int + coded_width: int + coded_height: int + layout: Union[int, CogFrameLayout] + is_key_frame: bool + temporal_offset: int + + +class _GrainGrainMetadataDict_cogcodedframe (_GrainGrainMetadataDict_cogcodedframe_MANDATORY, total=False): + unit_offsets: Sequence[int] + length: int + + +class CodedVideoGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['coded_video'] + cog_coded_frame: _GrainGrainMetadataDict_cogcodedframe + + +class _GrainGrainMetadataDict_eventpayload (TypedDict): + type: str + topic: str + data: List[EventGrainDatumDict] + + +class EventGrainGrainMetadataDict (_GrainGrainMetadataDict_common): + grain_type: Literal['event'] + event_payload: _GrainGrainMetadataDict_eventpayload + + +class _EmptyGrainMetadataDict_MANDATORY(TypedDict): + grain: EmptyGrainGrainMetadataDict + + +class _AudioGrainMetadataDict_MANDATORY(TypedDict): + grain: AudioGrainGrainMetadataDict + + +class _CodedAudioGrainMetadataDict_MANDATORY(TypedDict): + grain: CodedAudioGrainGrainMetadataDict + + +class _VideoGrainMetadataDict_MANDATORY(TypedDict): + grain: VideoGrainGrainMetadataDict + + +class _CodedVideoGrainMetadataDict_MANDATORY(TypedDict): + grain: CodedVideoGrainGrainMetadataDict + + +class _EventGrainMetadataDict_MANDATORY(TypedDict): + grain: EventGrainGrainMetadataDict + + +_GrainMetadataDict_OPTIONAL = TypedDict("_GrainMetadataDict_OPTIONAL", {"@_ns": str}, total=False) + + +class EmptyGrainMetadataDict(_EmptyGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +class AudioGrainMetadataDict(_AudioGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +class CodedAudioGrainMetadataDict(_CodedAudioGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +class VideoGrainMetadataDict(_VideoGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +class CodedVideoGrainMetadataDict(_CodedVideoGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +class EventGrainMetadataDict(_EventGrainMetadataDict_MANDATORY, _GrainMetadataDict_OPTIONAL): + pass + + +GrainMetadataDict = Union[ + EmptyGrainMetadataDict, + AudioGrainMetadataDict, + CodedAudioGrainMetadataDict, + VideoGrainMetadataDict, + CodedVideoGrainMetadataDict, + EventGrainMetadataDict] + + +# This is the type that defines what can go in a grain data element, there may be some corner cases not covered by this +GrainDataType = Union[SupportsBytes, bytes] + +GrainDataParameterType = Optional[Union[GrainDataType, Awaitable[Optional[GrainDataType]]]] diff --git a/mediagrains/utils/__init__.py b/mediagrains/utils/__init__.py index 72b7427..707e761 100644 --- a/mediagrains/utils/__init__.py +++ b/mediagrains/utils/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,9 +13,7 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - from .iobytes import IOBytes +from .grain_wrapper import GrainWrapper -__all__ = ["IOBytes"] +__all__ = ["IOBytes", "GrainWrapper"] diff --git a/mediagrains/utils/asyncbinaryio.py b/mediagrains/utils/asyncbinaryio.py new file mode 100644 index 0000000..884ec9d --- /dev/null +++ b/mediagrains/utils/asyncbinaryio.py @@ -0,0 +1,346 @@ +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""\ +An abstract base class for asynchronous equivalent of io.RawIOBase + +I haven't included all of the machinary in that other class, it seems more logical to simplify the +interface a little. +""" + +from abc import ABCMeta, abstractmethod +from io import SEEK_SET, SEEK_CUR + +from typing import Type, Union, Optional, IO, cast +from io import RawIOBase, UnsupportedOperation + +from asyncio import StreamReader, StreamWriter + + +class OpenAsyncBinaryIO(metaclass=ABCMeta): + async def read(self, size: int = -1) -> bytes: + if size == -1: + return await self.readall() + else: + while True: + b = bytearray(size) + s = await self.readinto(b) + if s is None: + continue + if s < size: + raise EOFError + return bytes(b) + + @abstractmethod + async def readinto(self, b: bytearray) -> Union[int, None]: ... + + @abstractmethod + async def readall(self) -> bytes: ... + + @abstractmethod + async def write(self, b: bytes) -> Optional[int]: ... + + @abstractmethod + async def truncate(self, s: Optional[int] = None) -> int: ... + + @abstractmethod + def tell(self) -> int: ... + + @abstractmethod + def seek(self, offset: int, whence: int = SEEK_SET): ... + + @abstractmethod + def seekable(self) -> bool: ... + + def seekable_forwards(self) -> bool: + return self.seekable() + + def seekable_backwards(self) -> bool: + return self.seekable() + + @abstractmethod + def readable(self) -> bool: ... + + @abstractmethod + def writable(self) -> bool: ... + + async def __open__(self) -> None: + "This coroutine should include any code that is to be run when the io stream is opened" + pass + + async def __close__(self) -> None: + "This coroutine should include any code that is to be run when the io stream is closed" + pass + + +class AsyncBinaryIO: + def __init__(self, cls: Type[OpenAsyncBinaryIO], *args, **kwargs): + self._inst = cls(*args, **kwargs) # type: ignore + + async def __aenter__(self) -> OpenAsyncBinaryIO: + await self._inst.__open__() + return self._inst + + async def __aexit__(self, *args, **kwargs) -> None: + await self._inst.__close__() + + +class OpenAsyncBytesIO(OpenAsyncBinaryIO): + def __init__(self, b: bytes): + self._buffer = bytearray(b) + self._pos = 0 + self._len = len(b) + + async def __open__(self) -> None: + "This coroutine should include any code that is to be run when the io stream is opened" + self._pos = 0 + + async def readinto(self, b: bytearray) -> int: + length = min(self._len - self._pos, len(b)) + if length > 0: + b[:length] = self._buffer[self._pos:self._pos + length] + self._pos += length + return length + else: + return 0 + + async def readall(self) -> bytes: + if self._pos >= 0 and self._pos < self._len: + return bytes(self._buffer[self._pos:self._len]) + else: + return bytes() + + async def write(self, b: bytes) -> int: + if self._pos < 0: + return 0 + + if self._pos + len(b) > len(self._buffer): + newbuf = bytearray(max(self._pos + len(b), 2*len(self._buffer))) + newbuf[:self._len] = self._buffer[:self._len] + self._buffer = newbuf + + length = len(b) + self._buffer[self._pos:self._pos + length] = b[:length] + self._pos += length + self._len = max(self._pos, self._len) + return length + + async def truncate(self, size: Optional[int] = None) -> int: + if size is not None: + self._len = size + else: + self._len = max(self._pos, 0) + + return self._len + + def tell(self) -> int: + return self._pos + + def seek(self, offset: int, whence: int = SEEK_SET): + if whence == SEEK_SET: + self._pos = offset + elif whence == SEEK_CUR: + self._pos += offset + else: + self._pos = self._len + offset + + def seekable(self) -> bool: + return True + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return True + + def getbuffer(self) -> bytearray: + return self._buffer[:self._len] + + def value(self) -> bytes: + return bytes(self._buffer[:self._len]) + + +class AsyncBytesIO(AsyncBinaryIO): + def __init__(self, b: bytes = b""): + super().__init__(cls=OpenAsyncBytesIO, b=b) + self._inst: OpenAsyncBytesIO + + def getbuffer(self) -> bytearray: + return self._inst.getbuffer() + + def value(self) -> bytes: + return self._inst.value() + + +class OpenAsyncFileWrapper(OpenAsyncBinaryIO): + def __init__(self, fp: IO[bytes]): + self.fp = cast(RawIOBase, fp) + + async def __open__(self) -> None: + # self.fp.__enter__() + pass + + async def __close__(self) -> None: + # self.fp.__exit__(None, None, None) + pass + + async def read(self, s: int = -1) -> bytes: + while True: + r = self.fp.read(s) + if r is not None: + return r + + async def readinto(self, b: bytearray) -> Optional[int]: + return self.fp.readinto(b) + + async def readall(self) -> bytes: + return self.fp.readall() + + async def write(self, b: bytes) -> Optional[int]: + return self.fp.write(b) + + async def truncate(self, size: Optional[int] = None) -> int: + return self.fp.truncate(size) + + def tell(self) -> int: + return self.fp.tell() + + def seek(self, offset: int, whence: int = SEEK_SET): + return self.fp.seek(offset, whence) + + def seekable(self) -> bool: + return self.fp.seekable() + + def readable(self) -> bool: + return self.fp.readable() + + def writable(self) -> bool: + return self.fp.writable() + + def getsync(self) -> IO[bytes]: + return cast(IO[bytes], self.fp) + + +class AsyncFileWrapper(AsyncBinaryIO): + def __init__(self, fp: IO[bytes]): + super().__init__(cls=OpenAsyncFileWrapper, fp=fp) + self._inst: OpenAsyncFileWrapper + self.fp = fp + + +class OpenAsyncStreamWrapper(OpenAsyncBinaryIO): + def __init__(self, reader: Optional[StreamReader] = None, writer: Optional[StreamWriter] = None): + self.reader = reader + self.writer = writer + self._pos = 0 + self._next_pos = 0 + + async def __open__(self) -> None: + self._pos = 0 + self._next_pos = 0 + + async def __close__(self) -> None: + if self.writer is not None: + self.writer.close() + + async def _align_pos(self): + if self._next_pos > self._pos: + if self.reader is not None: + await self.reader.read(self._next_pos - self._pos) + if self.writer is not None: + self.writer.write(bytes(self._next_pos - self._pos)) + self._pos = self._next_pos + + async def read(self, s: int = -1) -> bytes: + if self.reader is None: + raise UnsupportedOperation("Attempted to read from an output stream") + await self._align_pos() + d = await self.reader.read(s) + self._pos += len(d) + return d + + async def readinto(self, b: bytearray) -> Optional[int]: + if self.reader is None: + raise UnsupportedOperation("Attempted to read from an output stream") + await self._align_pos() + d = await self.reader.read(len(b)) + if d is None: + return 0 + else: + b[:len(d)] = d + self._pos += len(d) + return len(d) + + async def readall(self) -> bytes: + d = await self.read() + self._pos += len(d) + return d + + async def write(self, b: bytes) -> Optional[int]: + if self.writer is None: + raise UnsupportedOperation("Attempted to write to an input stream") + await self._align_pos() + self.writer.write(b) + await self.writer.drain() + self._pos += len(b) + return len(b) + + async def truncate(self, size: Optional[int] = None) -> int: + raise UnsupportedOperation("Cannot truncate a network stream") + + def tell(self) -> int: + return self._pos + + def seek(self, offset: int, whence: int = SEEK_SET): + next_pos = self._next_pos + if whence == SEEK_SET: + next_pos = offset + elif whence == SEEK_CUR: + next_pos += offset + else: + raise UnsupportedOperation("Cannot seek backwards") + if next_pos < self._pos: + raise UnsupportedOperation("Cannot seek backwards") + self._next_pos = next_pos + return self._next_pos + + def seekable(self) -> bool: + return False + + def readable(self) -> bool: + return (self.reader is not None) + + def writable(self) -> bool: + return (self.writer is not None) + + def seekable_backwards(self) -> bool: + return False + + def seekable_forwards(self) -> bool: + return True + + def getstream(self): + return (self.reader, self.writer) + + +class AsyncStreamWrapper(AsyncBinaryIO): + def __init__(self, reader: Optional[StreamReader] = None, writer: Optional[StreamWriter] = None): + super().__init__(cls=OpenAsyncStreamWrapper, reader=reader, writer=writer) + self._inst: OpenAsyncStreamWrapper + self.reader = reader + self.writer = writer + + def getstream(self): + return (self.reader, self.writer) diff --git a/mediagrains/utils/grain_wrapper.py b/mediagrains/utils/grain_wrapper.py new file mode 100644 index 0000000..75d168f --- /dev/null +++ b/mediagrains/utils/grain_wrapper.py @@ -0,0 +1,64 @@ +# +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""\ +Support for reading raw essence data and wrapping it in Grains, or reading Grains and returning essence data. +This can be used for tasks like piping the output of ffmpeg (or similar) into a GSF file. +""" +import typing +import copy + +from mediatimestamp.immutable import TimeRange + +from ..grain import GRAIN + + +class GrainWrapper(object): + """Raw input and wrap it in Grains""" + def __init__( + self, + template_grain: GRAIN, + input_data: typing.IO[bytes] + ): + """Set up the wrapper and the Grains that will be generated + + :param template_grain: A Grain to use as the template for wrapping the Grains read from the input source. Rate + and origin_timestamp should be set, along with the relevant metadata to make + `template_grain.expected_length` work. + :param input_data: An object to read video data from + """ + self.template_grain = template_grain + self.input_data = input_data + + self.frame_size = template_grain.expected_length + + def grains(self) -> typing.Iterator[GRAIN]: + """Generator that yields Grains read from the input given + + :yields: Grain objects read from the raw input supplied + """ + grain_timerange = TimeRange.from_start(self.template_grain.origin_timestamp) + + for timestamp in grain_timerange.at_rate(self.template_grain.rate): + new_grain = copy.deepcopy(self.template_grain) + new_grain.origin_timestamp = timestamp + + grain_data = self.input_data.read(self.frame_size) + + if grain_data: + new_grain.data = grain_data + yield new_grain + else: + break diff --git a/mediagrains/utils/iobytes.py b/mediagrains/utils/iobytes.py index d89815d..8ae825b 100644 --- a/mediagrains/utils/iobytes.py +++ b/mediagrains/utils/iobytes.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2019 British Broadcasting Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,14 +19,8 @@ bytes object, lazily loading as necessary. """ -from __future__ import print_function -from __future__ import absolute_import - -try: - from collections.abc import Sequence -except ImportError: - from collections import Sequence - +from collections.abc import Sequence +from typing import List __all__ = ["IOBytes"] @@ -47,7 +39,7 @@ class LazyLoader (object): transparently passed through to the stored object. """ - _attributes = [] + _attributes: List[str] = [] def __init__(self, loader): """ @@ -57,7 +49,7 @@ def __init__(self, loader): self._loader = loader def __getattribute__(self, attr): - if attr in (['_object', '_loader', '__repr__'] + type(self)._attributes): + if attr in (['_object', '_loader', '__repr__', '__class__'] + type(self)._attributes): return object.__getattribute__(self, attr) else: if object.__getattribute__(self, '_object') is None: diff --git a/mediagrains/utils/py.typed b/mediagrains/utils/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/mediagrains/utils/synchronise.py b/mediagrains/utils/synchronise.py new file mode 100644 index 0000000..310b095 --- /dev/null +++ b/mediagrains/utils/synchronise.py @@ -0,0 +1,175 @@ +# Copyright 2019 British Broadcasting Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""\ +Some utility functions for running synchronous and asynchronous code together. +""" + +from typing import TypeVar, Optional, Awaitable, Any, Generic, AsyncIterator, Iterator, Tuple +import asyncio +import threading +from inspect import iscoroutinefunction, isawaitable, isasyncgenfunction + +T = TypeVar('T') + + +class SynchronisationError(RuntimeError): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +def get_non_running_loop() -> Optional[asyncio.AbstractEventLoop]: + """If there is an existing runloop on this thread and it isn't running return it. + If there isn't one create one and return it. + If there is an existing running loop on this thread then return None. + """ + try: + loop = asyncio.get_event_loop() + except RuntimeError: + # In Python 3.6 trying to get an event loop when none can be automatically created is a an exception + return asyncio.new_event_loop() + else: + if loop.is_running(): + # There is already a running loop on this thread so we will need to spawn a new thread if we want to run one + return None + else: + # We have An existing event loop, but it is not running, so we can run it. + return loop + + +class ResultsType(Generic[T]): + def __init__(self): + self.rval: Optional[Tuple[T]] = None + self.exception: Optional[Exception] = None + + async def capture(self, a: Awaitable[T]) -> None: + try: + rval = await a + except Exception as e: + self.exception = e + else: + self.rval = (rval,) + + def restore(self) -> T: + if self.exception is not None: + raise self.exception + elif self.rval is None: + raise SynchronisationError("Expected a result but none was produced") + else: + return self.rval[0] + + +def run_awaitable_synchronously(f: Awaitable[T]) -> T: + """Runs an awaitable coroutine object as a synchronous call. + + Works from code running in a run-loop. + Works from code running outside a run-loop. + Works when there is a runloop already for this thread, but this code is not called from it. + """ + results = ResultsType[T]() + + def _run_awaitable_in_existing_run_loop(a: Awaitable[T], loop: asyncio.AbstractEventLoop) -> T: + loop.run_until_complete(results.capture(a)) + return results.restore() + + def _run_awaitable_in_new_thread(a: Awaitable[T]) -> T: + def __inner() -> None: + loop = asyncio.new_event_loop() + loop.run_until_complete(results.capture(a)) + loop.close() + + t = threading.Thread(target=__inner) + t.start() + t.join() + + return results.restore() + + loop = get_non_running_loop() + if loop is not None: + return _run_awaitable_in_existing_run_loop(f, loop) + else: + return _run_awaitable_in_new_thread(f) + + +def run_asyncgenerator_synchronously(gen: AsyncIterator[T]) -> Iterator[T]: + async def __get_next(gen): + return await gen.__anext__() + + if get_non_running_loop() is not None: + while True: + try: + yield run_awaitable_synchronously(__get_next(gen)) + except StopAsyncIteration: + return + else: + results = ResultsType[T]() + + def _run_generator_in_new_thread(gen: AsyncIterator[T]) -> Tuple[threading.Thread, threading.Event, threading.Event]: + agen_should_yield = threading.Event() + agen_has_yielded = threading.Event() + + def __inner() -> None: + async def _run_asyncgen_with_events(gen: AsyncIterator[T], agen_should_yield: threading.Event, agen_has_yielded: threading.Event) -> T: + try: + async for x in gen: + agen_should_yield.wait() + agen_should_yield.clear() + results.rval = (x,) + agen_has_yielded.set() + + agen_should_yield.wait() + agen_should_yield.clear() + raise StopAsyncIteration + finally: + agen_has_yielded.set() + + loop = asyncio.new_event_loop() + loop.run_until_complete(results.capture(_run_asyncgen_with_events(gen, agen_should_yield, agen_has_yielded))) + loop.close() + + t = threading.Thread(target=__inner) + t.daemon = True + t.start() + + return (t, agen_should_yield, agen_has_yielded) + + (t, agen_should_yield, agen_has_yielded) = _run_generator_in_new_thread(gen) + agen_should_yield.set() + agen_has_yielded.wait() + while t.is_alive(): + agen_has_yielded.clear() + try: + yield results.restore() + except StopAsyncIteration: + agen_should_yield.set() + return + agen_should_yield.set() + agen_has_yielded.wait() + + +class Synchronised(Generic[T]): + def __init__(self, other: T): + self._other = other + + def __getattr__(self, name: str) -> Any: + attr = getattr(self._other, name) + if iscoroutinefunction(attr): + return lambda *args, **kwargs: run_awaitable_synchronously(attr(*args, **kwargs)) + if isasyncgenfunction(attr): + return lambda *args, **kwargs: run_asyncgenerator_synchronously(attr(*args, **kwargs)) + elif isawaitable(attr): + return run_awaitable_synchronously(attr) + else: + return attr diff --git a/mediagrains_py36/psnr.py b/mediagrains_py36/psnr.py deleted file mode 100644 index e3f53ec..0000000 --- a/mediagrains_py36/psnr.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2019 British Broadcasting Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import math -import numpy as np - -from mediagrains.cogenums import COG_FRAME_IS_COMPRESSED, COG_FRAME_FORMAT_ACTIVE_BITS -from mediagrains.numpy import VideoGrain as numpy_VideoGrain, VIDEOGRAIN as numpy_VIDEOGRAIN - -__all__ = ["compute_psnr"] - - -def _compute_comp_mse(data_a, data_b): - """Compute MSE (Mean Squared Error) for video component. - - :param data_a: Data for component a - :param data_b: Data for component b - :returns: The MSE value - """ - return np.mean(np.square(np.subtract(data_a, data_b))) - - -def _compute_comp_psnr(data_a, data_b, max_val): - """Compute PSNR for video component. - - :param data_a: Data for component a - :param data_b: Data for component b - :param max_val: Maximum value for a component pixel - :returns: The PSNR - """ - mse = _compute_comp_mse(data_a, data_b) - if mse == 0: - return float('Inf') - else: - return 10.0 * math.log10((max_val**2)/mse) - - -def compute_psnr(grain_a, grain_b): - """Compute PSNR for video grains. - - :param grain_a: A VIDEOGRAIN - :param grain_b: A VIDEOGRAIN - :returns: A list of PSNR value for each video component - """ - if grain_a.grain_type != grain_b.grain_type or grain_a.grain_type != "video": - raise AttributeError("Invalid grain types") - if grain_a.width != grain_b.width or grain_a.height != grain_b.height: - raise AttributeError("Frame dimensions differ") - - if COG_FRAME_IS_COMPRESSED(grain_a.format): - raise NotImplementedError("Compressed video is not supported") - - if not isinstance(grain_a, numpy_VIDEOGRAIN): - grain_a = numpy_VideoGrain(grain_a) - if not isinstance(grain_b, numpy_VIDEOGRAIN): - grain_b = numpy_VideoGrain(grain_b) - - psnr = [] - max_val = (1 << COG_FRAME_FORMAT_ACTIVE_BITS(grain_a.format)) - 1 - for comp_data_a, comp_data_b in zip(grain_a.component_data, grain_b.component_data): - psnr.append(_compute_comp_psnr(comp_data_a, comp_data_b, max_val)) - - return psnr diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..16ef004 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[mypy-mediatimestamp.*,mediajson.*,numpy.*,frozendict.*,deprecated.*] +ignore_missing_imports = True diff --git a/setup.py b/setup.py index c366f8d..3dba6c8 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Copyright 2018 British Broadcasting Corporation # @@ -15,39 +15,41 @@ # limitations under the License. # -from __future__ import print_function - from setuptools import setup -from sys import version_info packages = { 'mediagrains': 'mediagrains', 'mediagrains.hypothesis': 'mediagrains/hypothesis', 'mediagrains.comparison': 'mediagrains/comparison', - 'mediagrains.utils': 'mediagrains/utils' + 'mediagrains.utils': 'mediagrains/utils', + 'mediagrains.asyncio': 'mediagrains/asyncio', + 'mediagrains.numpy': 'mediagrains/numpy', + 'mediagrains.tools': 'mediagrains/tools' } packages_required = [ - "mediatimestamp >= 1.2.0", - 'enum34 >= 1.1.6;python_version<"3.4"', - "six >= 1.10.0", + "mediatimestamp >= 1.3.0", "frozendict >= 1.2", - 'numpy >= 1.17.2;python_version>="3.6"', - 'numpy;python_version<"3.6"' + 'numpy >= 1.17.2', + "mediajson", + 'mypy', + 'deprecated >= 1.2.6', ] deps_required = [] - -if version_info[0] > 3 or (version_info[0] == 3 and version_info[1] >= 6): - packages['mediagrains_py36'] = 'mediagrains_py36' - packages['mediagrains_py36.asyncio'] = 'mediagrains_py36/asyncio' - - package_names = list(packages.keys()) +console_scripts = [ + 'wrap_video_in_gsf=mediagrains.tools:wrap_video_in_gsf', + 'wrap_audio_in_gsf=mediagrains.tools:wrap_audio_in_gsf', + 'extract_gsf_essence=mediagrains.tools:extract_gsf_essence', + 'gsf_probe=mediagrains.tools:gsf_probe' +] + setup(name="mediagrains", - version="2.6.0.post0", + version="2.7.0", + python_requires='>=3.6.0', description="Simple utility for grain-based media", url='https://github.com/bbc/rd-apmm-python-lib-mediagrains', author='James Weaver', @@ -55,8 +57,11 @@ license='Apache 2', packages=package_names, package_dir=packages, + package_data={name: ['py.typed'] for name in package_names}, install_requires=packages_required, - scripts=[], + entry_points={ + 'console_scripts': console_scripts + }, data_files=[], long_description=""" Simple python library for dealing with grain data in a python-native format. diff --git a/tests/fixtures.py b/tests/fixtures.py index 27953c6..c374eb7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -25,6 +24,11 @@ from copy import deepcopy +import asyncio +import warnings + +from functools import wraps + def pairs_of(strategy): return lists(strategy, min_size=2, max_size=2).map(tuple) @@ -61,3 +65,58 @@ def attribute_and_pairs_of_grains_of_type_differing_only_in_one_attribute(grain_ return grain_strat | grains(grain_type).flatmap(lambda g: tuples(just("data"), pairs_of(grains_from_template_with_data(g)))) else: return grain_strat + + +def suppress_deprecation_warnings(f): + @wraps(f) + def __inner(*args, **kwargs): + with warnings.catch_warnings(record=True) as warns: + r = f(*args, **kwargs) + + for w in warns: + if w.category != DeprecationWarning: + warnings.showwarning(w.message, w.category, w.filename, w.lineno) + + return r + + +def async_test(suppress_warnings): + def __outer(f): + @wraps(f) + def __inner(*args, **kwargs): + loop = asyncio.get_event_loop() + loop.set_debug(True) + E = None + warns = [] + + try: + with warnings.catch_warnings(record=True) as warns: + loop.run_until_complete(f(*args, **kwargs)) + + except AssertionError as e: + E = e + except Exception as e: + E = e + + runtime_warnings = [w for w in warns if w.category == RuntimeWarning] + + for w in (runtime_warnings if suppress_warnings else warns): + warnings.showwarning(w.message, + w.category, + w.filename, + w.lineno) + if E is None: + args[0].assertEqual(len(runtime_warnings), 0, + msg="asyncio subsystem generated warnings due to unawaited coroutines") + else: + raise E + + return __inner + + if callable(suppress_warnings): + # supress_warnings is actually f + f = suppress_warnings + suppress_warnings = False + return __outer(f) + else: + return __outer diff --git a/tests/test36_asyncio_gsf.py b/tests/test_asyncio_gsf.py similarity index 96% rename from tests/test36_asyncio_gsf.py rename to tests/test_asyncio_gsf.py index e48ac44..dcc6427 100644 --- a/tests/test36_asyncio_gsf.py +++ b/tests/test_asyncio_gsf.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2019 British Broadcasting Corporation @@ -16,13 +15,21 @@ # limitations under the License. # + +# +# The code these tests test is deprecated +# +# The tests remain until the code is removed +# + + from unittest import TestCase -import asyncio -import warnings import aiofiles from datetime import datetime from uuid import UUID +from fixtures import async_test + from mediagrains.asyncio import AsyncGSFDecoder, AsyncLazyLoaderUnloadedError, loads from mediagrains.grain import VIDEOGRAIN, AUDIOGRAIN, EVENTGRAIN, CODEDVIDEOGRAIN, CODEDAUDIOGRAIN from mediagrains.gsf import GSFDecodeError @@ -52,38 +59,8 @@ INTERLEAVED_DATA = f.read() -def async_test(f): - def __inner(*args, **kwargs): - loop = asyncio.get_event_loop() - loop.set_debug(True) - E = None - warns = [] - - try: - with warnings.catch_warnings(record=True) as warns: - loop.run_until_complete(f(*args, **kwargs)) - - except AssertionError as e: - E = e - except Exception as e: - E = e - - for w in warns: - warnings.showwarning(w.message, - w.category, - w.filename, - w.lineno) - if E is None: - args[0].assertEqual(len(warns), 0, - msg="asyncio subsystem generated warnings due to unawaited coroutines") - else: - raise E - - return __inner - - class TestAsyncGSFBlock (TestCase): - @async_test + @async_test(suppress_warnings=True) async def test_decode_headers(self): async with aiofiles.open('examples/video.gsf', 'rb') as video_data_stream: UUT = AsyncGSFDecoder(file_data=video_data_stream) @@ -94,7 +71,7 @@ async def test_decode_headers(self): self.assertEqual(len(head['segments']), 1) self.assertEqual(head['segments'][0]['id'], UUID('c6a3d3ff-74c0-446d-b59e-de1041f27e8a')) - @async_test + @async_test(suppress_warnings=True) async def test_generate_grains(self): """Test that the generator yields each grain""" async with aiofiles.open('examples/video.gsf', 'rb') as video_data_stream: @@ -108,7 +85,7 @@ async def test_generate_grains(self): self.assertEqual(10, grain_count) # There are 10 grains in the file - @async_test + @async_test(suppress_warnings=True) async def test_local_id_filtering(self): async with aiofiles.open('examples/interleaved.gsf', 'rb') as interleaved_data_stream: async with AsyncGSFDecoder(file_data=interleaved_data_stream) as UUT: @@ -136,7 +113,7 @@ async def test_local_id_filtering(self): self.assertEqual(grain.flow_id, UUID('2472f38e-3517-11e9-8da2-5065f34ed007')) self.assertEqual(local_id, 2) - @async_test + @async_test(suppress_warnings=True) async def test_lazy_loading(self): async with aiofiles.open('examples/video.gsf', 'rb') as video_data_stream: grains = [grain async for (grain, local_id) in AsyncGSFDecoder(file_data=video_data_stream).grains()] @@ -150,7 +127,7 @@ async def test_lazy_loading(self): class TestAsyncGSFLoads(TestCase): - @async_test + @async_test(suppress_warnings=True) async def test_loads_video(self): (head, segments) = await loads(VIDEO_DATA) @@ -197,7 +174,7 @@ async def test_loads_video(self): self.assertEqual(len(grain.data), grain.components[0].length + grain.components[1].length + grain.components[2].length) - @async_test + @async_test(suppress_warnings=True) async def test_loads_audio(self): (head, segments) = await loads(AUDIO_DATA) @@ -227,7 +204,7 @@ async def test_loads_audio(self): total_samples += grain.samples ots = start_ots + TimeOffset.from_count(total_samples, grain.sample_rate) - @async_test + @async_test(suppress_warnings=True) async def test_loads_coded_video(self): (head, segments) = await loads(CODED_VIDEO_DATA) @@ -269,14 +246,14 @@ async def test_loads_coded_video(self): self.assertEqual(grain.unit_offsets, unit_offsets[0][0]) unit_offsets.pop(0) - @async_test + @async_test(suppress_warnings=True) async def test_loads_rejects_incorrect_type_file(self): with self.assertRaises(GSFDecodeBadFileTypeError) as cm: await loads(b"POTATO23\x07\x00\x00\x00") self.assertEqual(cm.exception.offset, 0) self.assertEqual(cm.exception.filetype, "POTATO23") - @async_test + @async_test(suppress_warnings=True) async def test_loads_rejects_incorrect_version_file(self): with self.assertRaises(GSFDecodeBadVersionError) as cm: await loads(b"SSBBgrsg\x08\x00\x03\x00") @@ -284,20 +261,20 @@ async def test_loads_rejects_incorrect_version_file(self): self.assertEqual(cm.exception.major, 8) self.assertEqual(cm.exception.minor, 3) - @async_test + @async_test(suppress_warnings=True) async def test_loads_rejects_bad_head_tag(self): with self.assertRaises(GSFDecodeError) as cm: await loads(b"SSBBgrsg\x07\x00\x00\x00" + b"\xff\xff\xff\xff\x00\x00\x00\x00") self.assertEqual(cm.exception.offset, 12) - @async_test + @async_test(suppress_warnings=True) async def test_loads_raises_exception_without_head(self): with self.assertRaises(GSFDecodeError) as cm: await loads(b"SSBBgrsg\x07\x00\x00\x00") self.assertEqual(cm.exception.offset, 12) - @async_test + @async_test(suppress_warnings=True) async def test_loads_skips_unknown_block_before_head(self): (head, segments) = await loads(b"SSBBgrsg\x07\x00\x00\x00" + b"dumy\x08\x00\x00\x00" + @@ -310,7 +287,7 @@ async def test_loads_skips_unknown_block_before_head(self): self.assertEqual(head['segments'], []) self.assertEqual(head['tags'], []) - @async_test + @async_test(suppress_warnings=True) async def test_loads_skips_unknown_block_instead_of_segm(self): (head, segments) = await loads(b"SSBBgrsg\x07\x00\x00\x00" + b"head\x27\x00\x00\x00" + @@ -323,7 +300,7 @@ async def test_loads_skips_unknown_block_instead_of_segm(self): self.assertEqual(head['segments'], []) self.assertEqual(head['tags'], []) - @async_test + @async_test(suppress_warnings=True) async def test_loads_skips_unknown_block_before_segm(self): (head, segments) = await loads(b"SSBBgrsg\x07\x00\x00\x00" + (b"head\x49\x00\x00\x00" + @@ -344,7 +321,7 @@ async def test_loads_skips_unknown_block_before_segm(self): self.assertEqual(head['segments'][0]['count'], 0) self.assertEqual(head['tags'], []) - @async_test + @async_test(suppress_warnings=True) async def test_loads_raises_when_head_too_small(self): with self.assertRaises(GSFDecodeError) as cm: (head, segments) = await loads(b"SSBBgrsg\x07\x00\x00\x00" + @@ -359,7 +336,7 @@ async def test_loads_raises_when_head_too_small(self): self.assertEqual(cm.exception.offset, 51) - @async_test + @async_test(suppress_warnings=True) async def test_loads_raises_when_segm_too_small(self): with self.assertRaises(GSFDecodeError) as cm: (head, segments) = await loads(b"SSBBgrsg\x07\x00\x00\x00" + @@ -373,7 +350,7 @@ async def test_loads_raises_when_segm_too_small(self): self.assertEqual(cm.exception.offset, 77) - @async_test + @async_test(suppress_warnings=True) async def test_loads_decodes_tils(self): src_id = UUID('c707d64c-1596-11e8-a3fb-dca904824eec') flow_id = UUID('da78668a-1596-11e8-a577-dca904824eec') @@ -416,7 +393,7 @@ async def test_loads_decodes_tils(self): 'frame_rate_denominator': 1, 'drop_frame': False}}]) - @async_test + @async_test(suppress_warnings=True) async def test_loads_raises_when_grain_type_unknown(self): with self.assertRaises(GSFDecodeError) as cm: src_id = UUID('c707d64c-1596-11e8-a3fb-dca904824eec') @@ -443,7 +420,7 @@ async def test_loads_raises_when_grain_type_unknown(self): self.assertEqual(cm.exception.offset, 179) - @async_test + @async_test(suppress_warnings=True) async def test_loads_decodes_empty_grains(self): src_id = UUID('c707d64c-1596-11e8-a3fb-dca904824eec') flow_id = UUID('da78668a-1596-11e8-a577-dca904824eec') @@ -485,7 +462,7 @@ async def test_loads_decodes_empty_grains(self): self.assertEqual(segments[1][1].grain_type, "empty") self.assertIsNone(segments[1][1].data) - @async_test + @async_test(suppress_warnings=True) async def test_loads_coded_audio(self): (head, segments) = await loads(CODED_AUDIO_DATA) @@ -528,7 +505,7 @@ async def test_loads_coded_audio(self): total_samples += grain.samples ots = start_ots + TimeOffset.from_count(total_samples, grain.sample_rate) - @async_test + @async_test(suppress_warnings=True) async def test_loads_event(self): self.maxDiff = None (head, segments) = await loads(EVENT_DATA) diff --git a/tests/test_comparison.py b/tests/test_comparison.py index f2c31bd..7e06aaa 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -15,9 +14,6 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - import unittest from unittest import TestCase diff --git a/tests/test_grain.py b/tests/test_grain.py index db8f611..7a4410b 100644 --- a/tests/test_grain.py +++ b/tests/test_grain.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -15,7 +14,6 @@ # limitations under the License. # -from __future__ import print_function from unittest import TestCase import uuid from mediagrains import Grain, VideoGrain, AudioGrain, CodedVideoGrain, CodedAudioGrain, EventGrain @@ -26,13 +24,68 @@ import json from copy import copy, deepcopy +from fixtures import async_test + + +src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") +flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") +cts = Timestamp.from_tai_sec_nsec("417798915:0") +ots = Timestamp.from_tai_sec_nsec("417798915:5") +sts = Timestamp.from_tai_sec_nsec("417798915:10") + +VIDEOGRAIN_TEST_METADATA = { + "@_ns": "urn:x-ipstudio:ns:0.1", + "grain": { + "grain_type": "video", + "source_id": str(src_id), + "flow_id": str(flow_id), + "origin_timestamp": str(ots), + "sync_timestamp": str(sts), + "creation_timestamp": str(cts), + "rate": { + "numerator": 25, + "denominator": 1, + }, + "duration": { + "numerator": 1, + "denominator": 25, + }, + "cog_frame": { + "format": CogFrameFormat.S16_422_10BIT, + "width": 1920, + "height": 1080, + "layout": CogFrameLayout.FULL_FRAME, + "extension": 0, + "components": [ + { + 'stride': 4096, + 'width': 1920, + 'height': 1080, + 'offset': 0, + 'length': 4096*1080 + }, + { + 'stride': 2048, + 'width': 960, + 'height': 1080, + 'offset': 4096*1080, + 'length': 2048*1080 + }, + { + 'stride': 2048, + 'width': 960, + 'height': 1080, + 'offset': 4096*1080 + 2048*1080, + 'length': 2048*1080 + } + ] + } + }, +} + class TestGrain (TestCase): def test_empty_grain_creation(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(src_id, flow_id) @@ -63,12 +116,6 @@ def test_empty_grain_creation_with_missing_data(self): self.assertEqual(grain.creation_timestamp, cts) def test_empty_grain_creation_with_odd_data(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "grain": { "source_id": src_id, @@ -99,11 +146,6 @@ def test_empty_grain_creation_with_odd_data(self): self.assertEqual(grain.expected_length, 23) def test_empty_grain_creation_with_ots(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(src_id, flow_id, origin_timestamp=ots) @@ -120,12 +162,6 @@ def test_empty_grain_creation_with_ots(self): self.assertEqual(grain.timelabels, []) def test_empty_grain_creation_with_ots_and_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts) @@ -142,12 +178,6 @@ def test_empty_grain_creation_with_ots_and_sts(self): self.assertEqual(grain.timelabels, []) def test_empty_grain_castable_to_tuple(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts) @@ -161,12 +191,6 @@ def test_empty_grain_castable_to_tuple(self): self.assertIsInstance(tuple(grain[0]), tuple) def test_empty_grain_with_meta(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -223,41 +247,35 @@ def test_empty_grain_with_meta(self): self.assertEqual(repr(grain), "Grain({!r})".format(meta)) def test_empty_grain_setters(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts) - src_id = uuid.UUID("18d1a52e-0a67-11e8-ba57-776dc8ceabcb") - flow_id = uuid.UUID("1ed4cfb4-0a67-11e8-b803-733e0764879a") - cts = Timestamp.from_tai_sec_nsec("417798915:15") - ots = Timestamp.from_tai_sec_nsec("417798915:20") - sts = Timestamp.from_tai_sec_nsec("417798915:25") - grain_type = "potato" + new_src_id = uuid.UUID("18d1a52e-0a67-11e8-ba57-776dc8ceabcb") + new_flow_id = uuid.UUID("1ed4cfb4-0a67-11e8-b803-733e0764879a") + new_cts = Timestamp.from_tai_sec_nsec("417798915:15") + new_ots = Timestamp.from_tai_sec_nsec("417798915:20") + new_sts = Timestamp.from_tai_sec_nsec("417798915:25") + new_grain_type = "potato" - grain.grain_type = grain_type - self.assertEqual(grain.grain_type, grain_type) + grain.grain_type = new_grain_type + self.assertEqual(grain.grain_type, new_grain_type) - grain.source_id = src_id - self.assertEqual(grain.source_id, src_id) + grain.source_id = new_src_id + self.assertEqual(grain.source_id, new_src_id) - grain.flow_id = flow_id - self.assertEqual(grain.flow_id, flow_id) + grain.flow_id = new_flow_id + self.assertEqual(grain.flow_id, new_flow_id) - grain.origin_timestamp = ots - self.assertEqual(grain.origin_timestamp, ots) - self.assertEqual(grain.final_origin_timestamp(), ots) - self.assertEqual(grain.origin_timerange(), TimeRange.from_single_timestamp(ots)) + grain.origin_timestamp = new_ots + self.assertEqual(grain.origin_timestamp, new_ots) + self.assertEqual(grain.final_origin_timestamp(), new_ots) + self.assertEqual(grain.origin_timerange(), TimeRange.from_single_timestamp(new_ots)) - grain.sync_timestamp = sts - self.assertEqual(grain.sync_timestamp, sts) + grain.sync_timestamp = new_sts + self.assertEqual(grain.sync_timestamp, new_sts) - grain.creation_timestamp = cts - self.assertEqual(grain.creation_timestamp, cts) + grain.creation_timestamp = new_cts + self.assertEqual(grain.creation_timestamp, new_cts) grain.rate = 50 self.assertEqual(grain.rate, Fraction(50, 1)) @@ -319,12 +337,6 @@ def test_empty_grain_setters(self): } def test_video_grain_create_YUV422_10bit(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -411,12 +423,6 @@ def test_video_grain_create_sizes(self): (CogFrameFormat.v216, (1920*1080*4,)), (CogFrameFormat.UNKNOWN, ()), ]: - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=fmt, @@ -432,12 +438,6 @@ def test_video_grain_create_sizes(self): self.assertEqual(len(grain.data), offset) def test_video_component_setters(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -457,12 +457,6 @@ def test_video_component_setters(self): grain.components[0]['length'] = 17 self.assertEqual(grain.components[0].length, 17) - grain.components[0]['potato'] = 3 - self.assertIn('potato', grain.components[0]) - self.assertEqual(grain.components[0]['potato'], 3) - del grain.components[0]['potato'] - self.assertNotIn('potato', grain.components[0]) - grain.components.append({'stride': 1920, 'width': 1920, 'height': 1080, @@ -492,12 +486,6 @@ def test_video_component_setters(self): self.assertEqual(grain.components[0].length, 1920*1080) def test_video_grain_with_sparse_meta(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -529,12 +517,6 @@ def test_video_grain_with_sparse_meta(self): self.assertEqual(len(grain.components), 0) def test_video_grain_with_numeric_identifiers(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=0x2805, @@ -591,12 +573,6 @@ def test_video_grain_with_numeric_identifiers(self): 'length': 1920*1080*2}) def test_video_grain_setters(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -640,11 +616,6 @@ def test_video_grain_fails_with_no_metadata(self): VideoGrain(None) def test_video_grain_create_with_ots_and_no_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -657,10 +628,6 @@ def test_video_grain_create_with_ots_and_no_sts(self): self.assertEqual(grain.creation_timestamp, cts) def test_video_grain_create_with_no_ots_and_no_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -673,11 +640,6 @@ def test_video_grain_create_with_no_ots_and_no_sts(self): self.assertEqual(grain.creation_timestamp, cts) def test_videograin_meta_is_json_serialisable(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -686,62 +648,7 @@ def test_videograin_meta_is_json_serialisable(self): self.assertEqual(json.loads(json.dumps(grain.meta)), grain.meta) def test_grain_makes_videograin(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - - meta = { - "@_ns": "urn:x-ipstudio:ns:0.1", - "grain": { - "grain_type": "video", - "source_id": str(src_id), - "flow_id": str(flow_id), - "origin_timestamp": str(ots), - "sync_timestamp": str(sts), - "creation_timestamp": str(cts), - "rate": { - "numerator": 25, - "denominator": 1, - }, - "duration": { - "numerator": 1, - "denominator": 25, - }, - "cog_frame": { - "format": CogFrameFormat.S16_422_10BIT, - "width": 1920, - "height": 1080, - "layout": CogFrameLayout.FULL_FRAME, - "extension": 0, - "components": [ - { - 'stride': 4096, - 'width': 1920, - 'height': 1080, - 'offset': 0, - 'length': 4096*1080 - }, - { - 'stride': 2048, - 'width': 960, - 'height': 1080, - 'offset': 4096*1080, - 'length': 2048*1080 - }, - { - 'stride': 2048, - 'width': 960, - 'height': 1080, - 'offset': 4096*1080 + 2048*1080, - 'length': 2048*1080 - } - ] - } - }, - } - + meta = VIDEOGRAIN_TEST_METADATA data = bytearray(8192*1080) with mock.patch.object(Timestamp, "get_time", return_value=cts): @@ -753,61 +660,7 @@ def test_grain_makes_videograin(self): self.assertEqual(grain.data, data) def test_grain_makes_videograin_without_data(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - - meta = { - "@_ns": "urn:x-ipstudio:ns:0.1", - "grain": { - "grain_type": "video", - "source_id": str(src_id), - "flow_id": str(flow_id), - "origin_timestamp": str(ots), - "sync_timestamp": str(sts), - "creation_timestamp": str(cts), - "rate": { - "numerator": 25, - "denominator": 1, - }, - "duration": { - "numerator": 1, - "denominator": 25, - }, - "cog_frame": { - "format": CogFrameFormat.S16_422_10BIT, - "width": 1920, - "height": 1080, - "layout": CogFrameLayout.FULL_FRAME, - "extension": 0, - "components": [ - { - 'stride': 4096, - 'width': 1920, - 'height': 1080, - 'offset': 0, - 'length': 4096*1080 - }, - { - 'stride': 2048, - 'width': 960, - 'height': 1080, - 'offset': 4096*1080, - 'length': 2048*1080 - }, - { - 'stride': 2048, - 'width': 960, - 'height': 1080, - 'offset': 4096*1080 + 2048*1080, - 'length': 2048*1080 - } - ] - } - }, - } + meta = VIDEOGRAIN_TEST_METADATA with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = Grain(meta) @@ -818,11 +671,55 @@ def test_grain_makes_videograin_without_data(self): self.assertEqual(grain.length, 0) self.assertEqual(grain.expected_length, 8192*1080) - def test_video_grain_normalise(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - ots = Timestamp.from_tai_sec_nsec("417798915:5") + @async_test + async def test_videograin_with_async_data(self): + meta = VIDEOGRAIN_TEST_METADATA + + async def _get_data(): + _data = bytearray(8192*1080) + for n in range(0, 16): + _data[n] = n & 0xFF + return _data + + data_awaitable = _get_data() + expected_data = await _get_data() + with mock.patch.object(Timestamp, "get_time", return_value=cts): + grain = Grain(meta, data=data_awaitable) + + self.assertEqual(grain.grain_type, "video") + self.assertEqual(grain.format, CogFrameFormat.S16_422_10BIT) + self.assertEqual(grain.meta, meta) + self.assertIsNone(grain.data) + + self.assertEqual((await grain)[:16], expected_data[:16]) + self.assertEqual(grain.data[:16], expected_data[:16]) + + @async_test + async def test_videograin_with_async_data_as_acm(self): + meta = VIDEOGRAIN_TEST_METADATA + + async def _get_data(): + _data = bytearray(8192*1080) + for n in range(0, 16): + _data[n] = n & 0xFF + return _data + + data_awaitable = _get_data() + expected_data = await _get_data() + + with mock.patch.object(Timestamp, "get_time", return_value=cts): + grain = Grain(meta, data=data_awaitable) + + self.assertEqual(grain.grain_type, "video") + self.assertEqual(grain.format, CogFrameFormat.S16_422_10BIT) + self.assertEqual(grain.meta, meta) + self.assertIsNone(grain.data) + + async with grain as _grain: + self.assertEqual(_grain.data[:16], expected_data[:16]) + + def test_video_grain_normalise(self): with mock.patch.object(Timestamp, "get_time", return_value=ots): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, rate=Fraction(25, 1), @@ -841,12 +738,6 @@ def test_video_grain_normalise(self): TimeRange.from_single_timestamp(ots).normalise(25, 1)) def test_audio_grain_create_S16_PLANES(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_audio_format=CogAudioFormat.S16_PLANES, @@ -875,11 +766,6 @@ def test_audio_grain_create_S16_PLANES(self): self.assertEqual(repr(grain), "AudioGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_audio_grain_create_fills_in_missing_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(src_id, flow_id, origin_timestamp=ots, cog_audio_format=CogAudioFormat.S16_PLANES, @@ -907,10 +793,6 @@ def test_audio_grain_create_fills_in_missing_sts(self): self.assertEqual(repr(grain), "AudioGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_audio_grain_create_fills_in_missing_ots(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.S16_PLANES, @@ -942,12 +824,6 @@ def test_audio_grain_create_fails_with_no_params(self): AudioGrain(None) def test_audio_grain_create_all_formats(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - for (fmt, length) in [(CogAudioFormat.S16_PLANES, 1920*2*2), (CogAudioFormat.S16_PAIRS, 1920*2*2), (CogAudioFormat.S16_INTERLEAVED, 1920*2*2), @@ -976,8 +852,6 @@ def test_audio_grain_create_all_formats(self): def test_audio_grain_create_fills_in_missing_meta(self): meta = {} - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(meta) @@ -991,8 +865,6 @@ def test_audio_grain_create_fills_in_missing_meta(self): def test_audio_grain_setters(self): meta = {} - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(meta) @@ -1011,10 +883,6 @@ def test_audio_grain_setters(self): self.assertEqual(grain.sample_rate, 48000) def test_audiograin_meta_is_json_serialisable(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = AudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.S16_PLANES, @@ -1026,12 +894,6 @@ def test_audiograin_meta_is_json_serialisable(self): self.fail(msg="Json serialisation produces: {} which is not json deserialisable".format(json.dumps(grain.meta))) def test_grain_makes_audiograin(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -1069,10 +931,6 @@ def test_grain_makes_audiograin(self): self.assertEqual(grain.data, data) def test_audio_grain_normalise(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - ots = Timestamp.from_tai_sec_nsec("417798915:2") - with mock.patch.object(Timestamp, "get_time", return_value=ots): grain = AudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.S16_PLANES, @@ -1092,12 +950,6 @@ def test_audio_grain_normalise(self): TimeRange(ots, final_ts).normalise(48000, 1)) def test_coded_video_grain_create_VC2(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.VC2, @@ -1130,9 +982,7 @@ def test_coded_video_grain_create_VC2(self): self.assertEqual(repr(grain), "CodedVideoGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_coded_video_grain_create_fills_empty_meta(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = {} - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(meta) @@ -1153,7 +1003,6 @@ def test_coded_video_grain_create_fills_empty_meta(self): self.assertEqual(grain.unit_offsets, []) def test_coded_video_grain_create_corrects_numeric_data(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = { 'grain': { 'cog_coded_frame': { @@ -1183,12 +1032,6 @@ def test_coded_video_grain_create_corrects_numeric_data(self): self.assertEqual(grain.unit_offsets, []) def test_coded_video_grain_setters(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.VC2, @@ -1244,11 +1087,6 @@ def test_coded_video_grain_setters(self): self.assertEqual(grain.unit_offsets, []) def test_coded_video_grain_create_with_data(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") data = bytearray(500) with mock.patch.object(Timestamp, "get_time", return_value=cts): @@ -1262,11 +1100,6 @@ def test_coded_video_grain_create_with_data(self): self.assertEqual(len(grain.data), grain.length) def test_coded_video_grain_create_with_cts_and_ots(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(src_id, flow_id, origin_timestamp=ots, cog_frame_format=CogFrameFormat.VC2, @@ -1278,10 +1111,6 @@ def test_coded_video_grain_create_with_cts_and_ots(self): self.assertEqual(grain.sync_timestamp, ots) def test_coded_video_grain_create_with_cts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.VC2, @@ -1297,10 +1126,6 @@ def test_coded_video_grain_create_fails_with_empty(self): CodedVideoGrain(None) def test_coded_video_grain_meta_is_json_serialisable(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedVideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.VC2, @@ -1310,12 +1135,6 @@ def test_coded_video_grain_meta_is_json_serialisable(self): self.assertEqual(json.loads(json.dumps(grain.meta)), grain.meta) def test_grain_makes_codedvideograin(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -1356,10 +1175,6 @@ def test_grain_makes_codedvideograin(self): self.assertEqual(grain.data, data) def test_coded_video_grain_normalise(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=ots): grain = CodedVideoGrain(src_id, flow_id, origin_timestamp=ots, rate=Fraction(25, 1), @@ -1379,12 +1194,6 @@ def test_coded_video_grain_normalise(self): TimeRange.from_single_timestamp(ots).normalise(25, 1)) def test_coded_audio_grain_create_MP1(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_audio_format=CogAudioFormat.MP1, @@ -1420,11 +1229,6 @@ def test_coded_audio_grain_create_MP1(self): self.assertEqual(repr(grain), "CodedAudioGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_coded_audio_grain_create_without_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(src_id, flow_id, origin_timestamp=ots, cog_audio_format=CogAudioFormat.MP1, @@ -1460,10 +1264,6 @@ def test_coded_audio_grain_create_without_sts(self): self.assertEqual(repr(grain), "CodedAudioGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_coded_audio_grain_create_without_sts_or_ots(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.MP1, @@ -1499,7 +1299,6 @@ def test_coded_audio_grain_create_without_sts_or_ots(self): self.assertEqual(repr(grain), "CodedAudioGrain({!r},< binary data of length {} >)".format(grain.meta, len(grain.data))) def test_coded_audio_grain_create_fills_empty_meta(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = {} with mock.patch.object(Timestamp, "get_time", return_value=cts): @@ -1521,7 +1320,6 @@ def test_coded_audio_grain_create_fills_empty_meta(self): self.assertEqual(grain.length, 0) def test_coded_audio_grain_create_corrects_numeric_data(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = { 'grain': { 'cog_coded_audio': { @@ -1550,7 +1348,6 @@ def test_coded_audio_grain_create_corrects_numeric_data(self): def test_coded_audio_grain_setters(self): meta = {} - cts = Timestamp.from_tai_sec_nsec("417798915:0") with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(meta) @@ -1578,7 +1375,6 @@ def test_coded_audio_grain_setters(self): def test_coded_audio_grain_with_data(self): meta = {} data = bytearray(15360) - cts = Timestamp.from_tai_sec_nsec("417798915:0") with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(meta, data) @@ -1591,10 +1387,6 @@ def test_coded_audio_grain_raises_on_empty(self): CodedAudioGrain(None) def test_codedaudiograin_meta_is_json_serialisable(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = CodedAudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.MP1, @@ -1611,12 +1403,6 @@ def test_codedaudiograin_meta_is_json_serialisable(self): self.fail(msg="Json serialisation produces: {} which is not json deserialisable".format(json.dumps(grain.meta))) def test_grain_makes_codedaudiograin(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -1656,10 +1442,6 @@ def test_grain_makes_codedaudiograin(self): self.assertEqual(grain.data, data) def test_coded_audio_grain_normalise(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - ots = Timestamp.from_tai_sec_nsec("417798915:2") - with mock.patch.object(Timestamp, "get_time", return_value=ots): grain = CodedAudioGrain(src_id, flow_id, origin_timestamp=ots, cog_audio_format=CogAudioFormat.MP1, @@ -1684,12 +1466,6 @@ def test_coded_audio_grain_normalise(self): TimeRange(ots, final_ts).normalise(48000, 1)) def test_event_grain_create(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = EventGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, event_type='urn:x-ipstudio:format:event.query', topic='/dummy') @@ -1715,11 +1491,6 @@ def test_event_grain_create(self): self.assertEqual(repr(grain), "EventGrain({!r})".format(grain.meta)) def test_event_grain_create_without_sts(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = EventGrain(src_id, flow_id, origin_timestamp=ots, event_type='urn:x-ipstudio:format:event.query', topic='/dummy') @@ -1742,10 +1513,6 @@ def test_event_grain_create_without_sts(self): self.assertEqual(repr(grain), "EventGrain({!r})".format(grain.meta)) def test_event_grain_create_without_sts_or_ots(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = EventGrain(src_id, flow_id, event_type='urn:x-ipstudio:format:event.query', topic='/dummy') @@ -1768,7 +1535,6 @@ def test_event_grain_create_without_sts_or_ots(self): self.assertEqual(repr(grain), "EventGrain({!r})".format(grain.meta)) def test_event_grain_create_fills_in_empty_meta(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = {} with mock.patch.object(Timestamp, "get_time", return_value=cts): @@ -1792,12 +1558,6 @@ def test_event_grain_create_fails_on_None(self): EventGrain(None) def test_event_grain_create_from_meta_and_data(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - meta = { "@_ns": "urn:x-ipstudio:ns:0.1", "grain": { @@ -1834,7 +1594,7 @@ def test_event_grain_create_from_meta_and_data(self): }) with mock.patch.object(Timestamp, "get_time", return_value=cts): - grain = Grain(meta, data) + grain = Grain(meta, data.encode('utf-8')) self.assertEqual(grain.grain_type, "event") self.assertEqual(grain.source_id, src_id) @@ -1860,7 +1620,6 @@ def test_event_grain_create_from_meta_and_data(self): self.assertEqual(grain.event_data[1].post, 'bong') def test_event_grain_setters(self): - cts = Timestamp.from_tai_sec_nsec("417798915:0") meta = {} with mock.patch.object(Timestamp, "get_time", return_value=cts): @@ -1921,12 +1680,6 @@ def test_event_grain_setters(self): grain.data = bytearray(json.dumps({'potato': "masher"}).encode('utf-8')) def test_copy(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.S16_422_10BIT, @@ -1947,12 +1700,6 @@ def test_copy(self): self.assertEqual(grain.data[1], clone.data[1]) def test_deepcopy(self): - src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") - flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") - cts = Timestamp.from_tai_sec_nsec("417798915:0") - ots = Timestamp.from_tai_sec_nsec("417798915:5") - sts = Timestamp.from_tai_sec_nsec("417798915:10") - with mock.patch.object(Timestamp, "get_time", return_value=cts): grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, cog_frame_format=CogFrameFormat.S16_422_10BIT, diff --git a/tests/test_gsf.py b/tests/test_gsf.py index 67a2be5..886b05c 100644 --- a/tests/test_gsf.py +++ b/tests/test_gsf.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2018 British Broadcasting Corporation @@ -16,12 +15,11 @@ # limitations under the License. # -from __future__ import print_function from unittest import TestCase from uuid import UUID from mediagrains import Grain, VideoGrain, AudioGrain, CodedVideoGrain, CodedAudioGrain, EventGrain from mediagrains.grain import VIDEOGRAIN, AUDIOGRAIN, CODEDVIDEOGRAIN, CODEDAUDIOGRAIN, EVENTGRAIN -from mediagrains.gsf import loads, load, dumps, GSFEncoder, GSFDecoder, GSFBlock +from mediagrains.gsf import loads, load, dumps, GSFEncoder, GSFDecoder, AsyncGSFBlock, GrainDataLoadingMode from mediagrains.gsf import GSFDecodeError from mediagrains.gsf import GSFEncodeError from mediagrains.gsf import GSFDecodeBadVersionError @@ -32,14 +30,14 @@ from mediatimestamp.immutable import Timestamp, TimeOffset from datetime import datetime from fractions import Fraction -from six import PY2, BytesIO, int2byte +from io import BytesIO +from mediagrains.utils.asyncbinaryio import AsyncBytesIO from frozendict import frozendict from os import SEEK_SET -if PY2: - import mock -else: - from unittest import mock +from fixtures import suppress_deprecation_warnings, async_test + +from unittest import mock with open('examples/video.gsf', 'rb') as f: VIDEO_DATA = f.read() @@ -91,6 +89,42 @@ def test_dumps_no_grains(self): self.assertIn(1, segments) self.assertEqual(len(segments[1]), 0) + @async_test + async def test_async_encode_no_grains(self): + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f, + tags=[('potato', 'harvest')], + segments=[{'tags': [('upside', 'down')]}]): + pass + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(head['tags'], [('potato', 'harvest')]) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 0) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertNotIn(head['segments'][0]['id'], [head['id']]) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(head['segments'][0]['tags'], [('upside', 'down')]) + + if len(segments) > 0: + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), 0) + def test_dumps_videograin(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -138,6 +172,57 @@ def test_dumps_videograin(self): self.assertEqual(segments[1][0].data, grain.data) + @async_test + async def test_async_encode_videograin(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + for i in range(0, len(grain.data)): + grain.data[i] = i & 0xFF + grain.source_aspect_ratio = Fraction(16, 9) + grain.pixel_aspect_ratio = 1 + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grain(grain) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 1) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'video') + self.assertEqual(segments[1][0].format, CogFrameFormat.S16_422_10BIT) + self.assertEqual(segments[1][0].width, 1920) + self.assertEqual(segments[1][0].height, 1080) + self.assertEqual(segments[1][0].source_aspect_ratio, Fraction(16, 9)) + self.assertEqual(segments[1][0].pixel_aspect_ratio, Fraction(1, 1)) + + self.assertEqual(segments[1][0].data, grain.data) + def test_dumps_videograins(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -199,6 +284,71 @@ def test_dumps_videograins(self): self.assertEqual(segments[1][1].data, grain1.data) + @async_test + async def test_async_encode_videograins(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + grain1 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + for i in range(0, len(grain0.data)): + grain0.data[i] = i & 0xFF + for i in range(0, len(grain1.data)): + grain1.data[i] = 0xFF - (i & 0xFF) + grain0.source_aspect_ratio = Fraction(16, 9) + grain0.pixel_aspect_ratio = 1 + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'video') + self.assertEqual(segments[1][0].format, CogFrameFormat.S16_422_10BIT) + self.assertEqual(segments[1][0].width, 1920) + self.assertEqual(segments[1][0].height, 1080) + self.assertEqual(segments[1][0].source_aspect_ratio, Fraction(16, 9)) + self.assertEqual(segments[1][0].pixel_aspect_ratio, Fraction(1, 1)) + + self.assertEqual(segments[1][0].data, grain0.data) + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'video') + self.assertEqual(segments[1][1].format, CogFrameFormat.S16_422_10BIT) + self.assertEqual(segments[1][1].width, 1920) + self.assertEqual(segments[1][1].height, 1080) + self.assertIsNone(segments[1][1].source_aspect_ratio) + self.assertIsNone(segments[1][1].pixel_aspect_ratio) + + self.assertEqual(segments[1][1].data, grain1.data) + def test_dumps_audiograins(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -254,6 +404,65 @@ def test_dumps_audiograins(self): self.assertEqual(segments[1][1].data, grain1.data) + @async_test + async def test_async_encode_audiograins(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = AudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.S16_PLANES, samples=1920, sample_rate=48000) + grain1 = AudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.S16_PLANES, samples=1920, sample_rate=48000) + for i in range(0, len(grain0.data)): + grain0.data[i] = i & 0xFF + for i in range(0, len(grain1.data)): + grain1.data[i] = 0xFF - (i & 0xFF) + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'audio') + self.assertEqual(segments[1][0].format, CogAudioFormat.S16_PLANES) + self.assertEqual(segments[1][0].samples, 1920) + self.assertEqual(segments[1][0].sample_rate, 48000) + + self.assertEqual(segments[1][0].data, grain0.data) + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'audio') + self.assertEqual(segments[1][1].format, CogAudioFormat.S16_PLANES) + self.assertEqual(segments[1][1].samples, 1920) + self.assertEqual(segments[1][1].sample_rate, 48000) + + self.assertEqual(segments[1][1].data, grain1.data) + def test_dumps_codedvideograins(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -321,6 +530,77 @@ def test_dumps_codedvideograins(self): self.assertEqual(segments[1][1].data, grain1.data) + @async_test + async def test_async_encode_codedvideograins(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = CodedVideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.VC2, origin_width=1920, origin_height=1080, coded_width=1920, + coded_height=1088, is_key_frame=True, temporal_offset=-23, length=1024, unit_offsets=[5, 15, 105]) + grain1 = CodedVideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.VC2, origin_width=1920, origin_height=1080, coded_width=1920, + coded_height=1088, temporal_offset=17, length=256) + for i in range(0, len(grain0.data)): + grain0.data[i] = i & 0xFF + for i in range(0, len(grain1.data)): + grain1.data[i] = 0xFF - (i & 0xFF) + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'coded_video') + self.assertEqual(segments[1][0].format, CogFrameFormat.VC2) + self.assertEqual(segments[1][0].origin_width, 1920) + self.assertEqual(segments[1][0].origin_height, 1080) + self.assertEqual(segments[1][0].coded_width, 1920) + self.assertEqual(segments[1][0].coded_height, 1088) + self.assertEqual(segments[1][0].temporal_offset, -23) + self.assertEqual(segments[1][0].unit_offsets, [5, 15, 105]) + self.assertTrue(segments[1][0].is_key_frame) + + self.assertEqual(segments[1][0].data, grain0.data) + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'coded_video') + self.assertEqual(segments[1][1].format, CogFrameFormat.VC2) + self.assertEqual(segments[1][1].origin_width, 1920) + self.assertEqual(segments[1][1].origin_height, 1080) + self.assertEqual(segments[1][1].coded_width, 1920) + self.assertEqual(segments[1][1].coded_height, 1088) + self.assertEqual(segments[1][1].temporal_offset, 17) + self.assertEqual(segments[1][1].unit_offsets, []) + self.assertFalse(segments[1][1].is_key_frame) + + self.assertEqual(segments[1][1].data, grain1.data) + def test_dumps_codedaudiograins(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -380,6 +660,69 @@ def test_dumps_codedaudiograins(self): self.assertEqual(segments[1][1].data, grain1.data) + @async_test + async def test_async_encode_codedaudiograins(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = CodedAudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.AAC, samples=1920, sample_rate=48000, priming=23, remainder=17, length=1024) + grain1 = CodedAudioGrain(src_id, flow_id, cog_audio_format=CogAudioFormat.AAC, samples=1920, sample_rate=48000, priming=5, remainder=104, length=1500) + for i in range(0, len(grain0.data)): + grain0.data[i] = i & 0xFF + for i in range(0, len(grain1.data)): + grain1.data[i] = 0xFF - (i & 0xFF) + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'coded_audio') + self.assertEqual(segments[1][0].format, CogAudioFormat.AAC) + self.assertEqual(segments[1][0].samples, 1920) + self.assertEqual(segments[1][0].sample_rate, 48000) + self.assertEqual(segments[1][0].priming, 23) + self.assertEqual(segments[1][0].remainder, 17) + + self.assertEqual(segments[1][0].data, grain0.data) + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'coded_audio') + self.assertEqual(segments[1][1].format, CogAudioFormat.AAC) + self.assertEqual(segments[1][1].samples, 1920) + self.assertEqual(segments[1][1].sample_rate, 48000) + self.assertEqual(segments[1][1].priming, 5) + self.assertEqual(segments[1][1].remainder, 104) + + self.assertEqual(segments[1][1].data, grain1.data) + def test_dumps_eventgrains(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -431,15 +774,140 @@ def test_dumps_eventgrains(self): self.assertEqual(segments[1][1].source_id, src_id) self.assertEqual(segments[1][1].flow_id, flow_id) - self.assertEqual(segments[1][1].grain_type, 'event') - self.assertEqual(segments[1][1].event_type, "urn:x-testing:clever/type") - self.assertEqual(segments[1][1].topic, "/inu") - self.assertEqual(len(segments[1][1].event_data), 1) - self.assertEqual(segments[1][1].event_data[0].path, "/sukimono") - self.assertEqual(segments[1][1].event_data[0].pre, "da") - self.assertIsNone(segments[1][1].event_data[0].post) + self.assertEqual(segments[1][1].grain_type, 'event') + self.assertEqual(segments[1][1].event_type, "urn:x-testing:clever/type") + self.assertEqual(segments[1][1].topic, "/inu") + self.assertEqual(len(segments[1][1].event_data), 1) + self.assertEqual(segments[1][1].event_data[0].path, "/sukimono") + self.assertEqual(segments[1][1].event_data[0].pre, "da") + self.assertIsNone(segments[1][1].event_data[0].post) + + @async_test + async def test_async_encode_eventgrains(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = EventGrain(src_id, flow_id) + grain0.event_type = "urn:x-testing:stupid/type" + grain0.topic = "/watashi" + grain0.append("/inu", post="desu") + grain1 = EventGrain(src_id, flow_id) + grain1.event_type = "urn:x-testing:clever/type" + grain1.topic = "/inu" + grain1.append("/sukimono", pre="da") + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'event') + self.assertEqual(segments[1][0].event_type, "urn:x-testing:stupid/type") + self.assertEqual(segments[1][0].topic, "/watashi") + self.assertEqual(len(segments[1][0].event_data), 1) + self.assertEqual(segments[1][0].event_data[0].path, "/inu") + self.assertIsNone(segments[1][0].event_data[0].pre) + self.assertEqual(segments[1][0].event_data[0].post, "desu") + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'event') + self.assertEqual(segments[1][1].event_type, "urn:x-testing:clever/type") + self.assertEqual(segments[1][1].topic, "/inu") + self.assertEqual(len(segments[1][1].event_data), 1) + self.assertEqual(segments[1][1].event_data[0].path, "/sukimono") + self.assertEqual(segments[1][1].event_data[0].pre, "da") + self.assertIsNone(segments[1][1].event_data[0].post) + + def test_dumps_emptygrains(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = Grain(src_id, flow_id) + grain0.timelabels = [{ + 'tag': 'tiggle', + 'timelabel': { + 'frames_since_midnight': 7, + 'frame_rate_numerator': 300, + 'frame_rate_denominator': 1, + 'drop_frame': False + } + }] + grain1 = Grain(src_id, flow_id) + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + (head, segments) = loads(dumps([grain0, grain1])) + + self.assertIn('id', head) + self.assertIn(head['id'], uuids) + self.assertIn('tags', head) + self.assertEqual(len(head['tags']), 0) + self.assertIn('created', head) + self.assertEqual(head['created'], created) + self.assertIn('segments', head) + self.assertEqual(len(head['segments']), 1) + self.assertIn('count', head['segments'][0]) + self.assertEqual(head['segments'][0]['count'], 2) + self.assertIn('local_id', head['segments'][0]) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertIn('id', head['segments'][0]) + self.assertIn(head['segments'][0]['id'], uuids) + self.assertIn('tags', head['segments'][0]) + self.assertEqual(len(head['segments'][0]['tags']), 0) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), head['segments'][0]['count']) + + self.assertEqual(segments[1][0].source_id, src_id) + self.assertEqual(segments[1][0].flow_id, flow_id) + self.assertEqual(segments[1][0].grain_type, 'empty') + self.assertEqual(segments[1][0].timelabels, [{ + 'tag': 'tiggle', + 'timelabel': { + 'frames_since_midnight': 7, + 'frame_rate_numerator': 300, + 'frame_rate_denominator': 1, + 'drop_frame': False + } + }]) + self.assertIsNone(segments[1][0].data) + + self.assertEqual(segments[1][1].source_id, src_id) + self.assertEqual(segments[1][1].flow_id, flow_id) + self.assertEqual(segments[1][1].grain_type, 'empty') + self.assertIsNone(segments[1][1].data) - def test_dumps_emptygrains(self): + @async_test + async def test_async_encode_emptygrains(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') grain0 = Grain(src_id, flow_id) @@ -458,7 +926,10 @@ def test_dumps_emptygrains(self): created = datetime(1983, 3, 29, 15, 15) with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): - (head, segments) = loads(dumps([grain0, grain1])) + f = AsyncBytesIO() + async with GSFEncoder(f) as enc: + await enc.add_grains([grain0, grain1]) + (head, segments) = loads(f.value()) self.assertIn('id', head) self.assertIn(head['id'], uuids) @@ -513,7 +984,23 @@ def test_dumps_invalidgrains(self): with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): (head, segments) = loads(dumps([grain])) - def test_dump_progressively(self): + @async_test + async def test_async_encode_invalidgrains(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain = Grain(src_id, flow_id) + grain.grain_type = "invalid" + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + with self.assertRaises(GSFEncodeError): + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + async with GSFEncoder(AsyncBytesIO()) as enc: + await enc.add_grains([grain]) + + @suppress_deprecation_warnings + def test_dump_progressively__deprecated(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') grain0 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) @@ -553,6 +1040,85 @@ def test_dump_progressively(self): self.assertEqual(len(segments2[1]), 2) self.assertEqual(len(segments3[1]), 2) + def test_dump_progressively(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + grain1 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + + file = BytesIO() + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + enc = GSFEncoder(file, streaming=True) + enc.add_segment() + self.assertEqual(len(file.getvalue()), 0) + with enc: + dump0 = file.getvalue() + (head0, segments0) = loads(dump0) + enc.add_grain(grain0) + dump1 = file.getvalue() + (head1, segments1) = loads(dump1) + enc.add_grain(grain1, segment_local_id=1) + dump2 = file.getvalue() + (head2, segments2) = loads(dump2) + dump3 = file.getvalue() + (head3, segments3) = loads(dump3) + + self.assertEqual(head0['segments'][0]['count'], -1) + self.assertEqual(head1['segments'][0]['count'], -1) + self.assertEqual(head2['segments'][0]['count'], -1) + self.assertEqual(head3['segments'][0]['count'], 2) + + if 1 in segments0: + self.assertEqual(len(segments0[1]), 0) + self.assertEqual(len(segments1[1]), 1) + self.assertEqual(len(segments2[1]), 2) + self.assertEqual(len(segments3[1]), 2) + + @async_test + async def test_async_encode_progressively(self): + src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') + flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') + grain0 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + grain1 = VideoGrain(src_id, flow_id, cog_frame_format=CogFrameFormat.S16_422_10BIT, width=1920, height=1080) + + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + + file = AsyncBytesIO() + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + enc = GSFEncoder(file, streaming=True, segments=[{}]) + self.assertEqual(len(file.value()), 0) + async with enc as enc: + dump0 = file.value() + (head0, segments0) = loads(dump0) + await enc.add_grain(grain0) + dump1 = file.value() + (head1, segments1) = loads(dump1) + await enc.add_grain(grain1, segment_local_id=1) + dump2 = file.value() + (head2, segments2) = loads(dump2) + dump3 = file.value() + (head3, segments3) = loads(dump3) + + self.assertEqual(head0['segments'][0]['count'], -1) + self.assertEqual(head1['segments'][0]['count'], -1) + self.assertEqual(head2['segments'][0]['count'], -1) + self.assertEqual(head3['segments'][0]['count'], 2) + + if 1 in segments0: + self.assertEqual(len(segments0[1]), 0) + self.assertEqual(len(segments1[1]), 1) + self.assertEqual(len(segments2[1]), 2) + self.assertEqual(len(segments3[1]), 2) + + @suppress_deprecation_warnings def test_end_dump_without_start_does_nothing(self): uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] @@ -605,7 +1171,8 @@ def test_encoder_access_methods(self): self.assertIsInstance(enc.segments, frozendict) self.assertEqual(enc.segments[1].tags, (('rainbow', 'dash'),)) - def test_encoder_raises_when_adding_to_active_encode(self): + @suppress_deprecation_warnings + def test_encoder_raises_when_adding_to_active_encode__deprecated(self): uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] created = datetime(1983, 3, 29, 15, 15) @@ -630,6 +1197,30 @@ def test_encoder_raises_when_adding_to_active_encode(self): with self.assertRaises(GSFEncodeAddToActiveDump): seg.add_tag('upside', 'down') + def test_encoder_raises_when_adding_to_active_encode(self): + uuids = [UUID('7920b394-1565-11e8-86e0-8b42d4647ba8'), + UUID('80af875c-1565-11e8-8f44-87ef081b48cd')] + created = datetime(1983, 3, 29, 15, 15) + file = BytesIO() + with mock.patch('mediagrains.gsf.datetime', side_effect=datetime, now=mock.MagicMock(return_value=created)): + with mock.patch('mediagrains.gsf.uuid1', side_effect=uuids): + enc = GSFEncoder(file, tags=[('potato', 'harvest')], streaming=True) + seg = enc.add_segment(tags=[('rainbow', 'dash')]) + + with self.assertRaises(GSFEncodeError): + enc.add_segment(local_id=1) + + with self.assertRaises(GSFEncodeError): + enc.add_segment(tags=[None]) + + with enc: + with self.assertRaises(GSFEncodeAddToActiveDump): + enc.add_tag('upside', 'down') + with self.assertRaises(GSFEncodeAddToActiveDump): + enc.add_segment() + with self.assertRaises(GSFEncodeAddToActiveDump): + seg.add_tag('upside', 'down') + def test_encoder_can_add_grains_to_nonexistent_segment(self): src_id = UUID('e14e9d58-1567-11e8-8dd3-831a068eb034') flow_id = UUID('ee1eed58-1567-11e8-a971-3b901a2dd8ab') @@ -650,115 +1241,141 @@ def test_encoder_can_add_grains_to_nonexistent_segment(self): class TestGSFBlock(TestCase): """Test the GSF decoder block handler correctly parses various types""" - def test_read_uint(self): + @async_test + async def test_read_uint(self): test_number = 4132 test_data = b"\x24\x10\x00\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_number, UUT.read_uint(4)) + self.assertEqual(test_number, await UUT.read_uint(4)) - def test_read_bool(self): + @async_test + async def test_read_bool(self): test_data = b"\x00\x01\x02" # False, True (0x01 != 0), True (0x02 != 0) - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertFalse(UUT.read_bool()) - self.assertTrue(UUT.read_bool()) - self.assertTrue(UUT.read_bool()) + self.assertFalse(await UUT.read_bool()) + self.assertTrue(await UUT.read_bool()) + self.assertTrue(await UUT.read_bool()) - def test_read_sint(self): + @async_test + async def test_read_sint(self): test_number = -12856 test_data = b"\xC8\xCD\xFF" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_number, UUT.read_sint(3)) + self.assertEqual(test_number, await UUT.read_sint(3)) - def test_read_string(self): + @async_test + async def test_read_string(self): """Test we can read a string, with Unicode characters""" test_string = u"Strings😁✔" test_data = b"Strings\xf0\x9f\x98\x81\xe2\x9c\x94" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_string, UUT.read_string(14)) + self.assertEqual(test_string, await UUT.read_string(14)) - def test_read_varstring(self): + @async_test + async def test_read_varstring(self): test_string = u"Strings😁✔" test_data = b"\x0e\x00Strings\xf0\x9f\x98\x81\xe2\x9c\x94" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_string, UUT.read_varstring()) + self.assertEqual(test_string, await UUT.read_varstring()) - def test_read_uuid(self): + @async_test + async def test_read_uuid(self): test_uuid = UUID("b06c65c8-51ac-4ad1-a839-2ef37107cc16") test_data = b"\xb0\x6c\x65\xc8\x51\xac\x4a\xd1\xa8\x39\x2e\xf3\x71\x07\xcc\x16" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_uuid, UUT.read_uuid()) + self.assertEqual(test_uuid, await UUT.read_uuid()) - def test_read_timestamp(self): + @async_test + async def test_read_timestamp(self): test_timestamp = datetime(2018, 9, 8, 16, 0, 0) test_data = b"\xe2\x07\x09\x08\x10\x00\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_timestamp, UUT.read_timestamp()) + self.assertEqual(test_timestamp, await UUT.read_timestamp()) - def test_read_ippts(self): + @async_test + async def test_read_ippts(self): test_timestamp = Timestamp(1536422400, 500) test_data = b"\x00\xf2\x93\x5b\x00\x00\xf4\x01\x00\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_timestamp, UUT.read_ippts()) + self.assertEqual(test_timestamp, await UUT.read_ippts()) - def test_read_rational(self): + @async_test + async def test_read_rational(self): test_fraction = Fraction(4, 3) test_data = b"\x04\x00\x00\x00\x03\x00\x00\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(test_fraction, UUT.read_rational()) + self.assertEqual(test_fraction, await UUT.read_rational()) - def test_read_rational_zero_denominator(self): + @async_test + async def test_read_rational_zero_denominator(self): """Ensure the reader considers a Rational with zero denominator to be 0, not an error""" test_data = b"\x04\x00\x00\x00\x00\x00\x00\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - self.assertEqual(Fraction(0), UUT.read_rational()) + self.assertEqual(Fraction(0), await UUT.read_rational()) - def test_read_uint_past_eof(self): + @async_test + async def test_read_uint_past_eof(self): """read_uint calls read() directly - test it raises EOFError correctly""" test_data = b"\x04\x00" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - with self.assertRaises(EOFError): - UUT.read_uint(4) + with self.assertRaises(EOFError): + await UUT.read_uint(4) - def test_read_string_past_eof(self): + @async_test + async def test_read_string_past_eof(self): """read_string() calls read() directly - test it raises EOFError correctly""" test_data = b"Strin" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - with self.assertRaises(EOFError): - UUT.read_string(6) + with self.assertRaises(EOFError): + await UUT.read_string(6) - def test_read_uuid_past_eof(self): + @async_test + async def test_read_uuid_past_eof(self): """read_uuid() calls read() directly - test it raises EOFError correctly""" test_data = b"\xb0\x6c\x65\xc8\x51\xac\x4a\xd1\xa8\x39\x2e" - UUT = GSFBlock(BytesIO(test_data)) + async with AsyncBytesIO(test_data) as fp: + UUT = AsyncGSFBlock(fp) - with self.assertRaises(EOFError): - UUT.read_uuid() + with self.assertRaises(EOFError): + await UUT.read_uuid() def _make_sample_stream(self, post_child_len=0): """Generate a stream of sample blocks for testing the context manager @@ -771,13 +1388,13 @@ def _make_sample_stream(self, post_child_len=0): blok (8 bytes) :param post_child_len: Number of bytes to include after last child block - must be <256 - :returns: BytesIO containing some blocks + :returns: AsyncBytesIO containing some blocks """ first_block_length = 28 + post_child_len test_stream = BytesIO() test_stream.write(b"blok") - test_stream.write(int2byte(first_block_length)) + test_stream.write(bytes((first_block_length,))) test_stream.write(b"\x00\x00\x00") test_stream.write(b"chil\x0c\x00\x00\x00\x08\x09\x0a\x0b") test_stream.write(b"chil\x08\x00\x00\x00") @@ -789,113 +1406,208 @@ def _make_sample_stream(self, post_child_len=0): test_stream.seek(0, SEEK_SET) - return test_stream + return AsyncBytesIO(test_stream.getvalue()) - def test_block_start_recorded(self): - """Test that a GSFBlock records the block start point""" - test_stream = self._make_sample_stream() - test_stream.seek(28, SEEK_SET) + @async_test + async def test_block_start_recorded(self): + """Test that a AsyncGSFBlock records the block start point""" + async with self._make_sample_stream() as test_stream: + test_stream.seek(28, SEEK_SET) - UUT = GSFBlock(test_stream) - self.assertEqual(28, UUT.block_start) + UUT = AsyncGSFBlock(test_stream) + self.assertEqual(28, UUT.block_start) - def test_contextmanager_read_tag_size(self): + @async_test + async def test_contextmanager_read_tag_size(self): """Test that the block tag and size are read when used in a context manager""" - test_stream = self._make_sample_stream() + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + self.assertEqual("blok", UUT.tag) + self.assertEqual(28, UUT.size) + + @async_test + async def test_contextmanager_skips_unwanted_blocks(self): + """Test that AsyncGSFBlock() seeks over unwanted blocks""" + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream, want_tag="blk2") as UUT: + self.assertEqual("blk2", UUT.tag) + + # blok is 28 bytes long, we should skip it, plus 8 bytes of blk2 + self.assertEqual(28 + 8, test_stream.tell()) + + @async_test + async def test_contextmanager_errors_unwanted_blocks(self): + """Test that AsyncGSFBlock() raises GSFDecodeError when finding an unwanted block""" + async with self._make_sample_stream() as test_stream: + with self.assertRaises(GSFDecodeError): + async with AsyncGSFBlock(test_stream, want_tag="chil", raise_on_wrong_tag=True): + pass - with GSFBlock(test_stream) as UUT: - self.assertEqual("blok", UUT.tag) - self.assertEqual(28, UUT.size) + @async_test + async def test_contextmanager_seeks_on_exit(self): + """Test that the context manager seeks to the end of a block on exit""" + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream): + pass - def test_contextmanager_skips_unwanted_blocks(self): - """Test that GSFBlock() seeks over unwanted blocks""" - test_stream = self._make_sample_stream() + # First block is 28 bytes long, so we should be zero-index position 28 afterwards + self.assertEqual(28, test_stream.tell()) - with GSFBlock(test_stream, want_tag="blk2") as UUT: - self.assertEqual("blk2", UUT.tag) + @async_test + async def test_contextmanager_get_remaining(self): + """Test the context manager gets the number of bytes left in the block correctly""" + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + await UUT.read_uint(4) # Use a read to skip ahead a bit + self.assertEqual(28 - 8 - 4, UUT.get_remaining()) # Block was 28 bytes, 8 bytes header, 4 bytes read_uint - # blok is 28 bytes long, we should skip it, plus 8 bytes of blk2 - self.assertEqual(28 + 8, test_stream.tell()) + @async_test + async def test_contextmanager_has_child(self): + """Test the context manager detects whether another child is present correctly""" + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + self.assertTrue(UUT.has_child_block()) + await test_stream.read(12) + self.assertTrue(UUT.has_child_block()) + await test_stream.read(8) + self.assertFalse(UUT.has_child_block()) + + @async_test + async def test_contextmanager_has_child_strict_blocks(self): + """Ensure that when strict mode is enabled, has_child errors on partial blocks""" + async with self._make_sample_stream(post_child_len=4) as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + await test_stream.read(12 + 8) # Read to the end of the child blocks - def test_contextmanager_errors_unwanted_blocks(self): - """Test that GSFBlock() raises GSFDecodeError when finding an unwanted block""" - test_stream = self._make_sample_stream() + with self.assertRaises(GSFDecodeError): + UUT.has_child_block(strict_blocks=True) - with self.assertRaises(GSFDecodeError): - with GSFBlock(test_stream, want_tag="chil", raise_on_wrong_tag=True): - pass + @async_test + async def test_contextmanager_has_child_no_strict_blocks(self): + """Ensure that when strict mode is off, has_child doesn't error on partial blocks""" + async with self._make_sample_stream(post_child_len=4) as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + await test_stream.read(12 + 8) # Read to the end of the child blocks - def test_contextmanager_seeks_on_exit(self): - """Test that the context manager seeks to the end of a block on exit""" - test_stream = self._make_sample_stream() + self.assertFalse(UUT.has_child_block(strict_blocks=False)) - with GSFBlock(test_stream): - pass + @async_test + async def test_contextmanager_child_blocks_generator(self): + """Ensure the child blocks generator returns a block, and seeks afterwards""" + async with self._make_sample_stream() as test_stream: + async with AsyncGSFBlock(test_stream) as UUT: + loop_count = 0 + child_bytes_consumed = 0 + async for block in UUT.child_blocks(): + child_bytes_consumed += block.size + loop_count += 1 - # First block is 28 bytes long, so we should be zero-index position 28 afterwards - self.assertEqual(28, test_stream.tell()) + # Did we get both child blocks? + self.assertEqual(2, loop_count) - def test_contextmanager_get_remaining(self): - """Test the context manager gets the number of bytes left in the block correctly""" - test_stream = self._make_sample_stream() + # Did we seek on exit from each loop iteration + self.assertEqual(child_bytes_consumed + UUT.block_start + 8, test_stream.tell()) - with GSFBlock(test_stream) as UUT: - UUT.read_uint(4) # Use a read to skip ahead a bit - self.assertEqual(28 - 8 - 4, UUT.get_remaining()) # Block was 28 bytes, 8 bytes header, 4 bytes read_uint - def test_contextmanager_has_child(self): - """Test the context manager detects whether another child is present correctly""" - test_stream = self._make_sample_stream() +class TestGSFDecoder(TestCase): + """Tests for the GSFDecoder in its more object-oriented mode + + Note that very little testing of the decoded data happens here, that's handled by TestGSFLoads() + """ + def test_decode_headers(self): + video_data_stream = BytesIO(VIDEO_DATA) - with GSFBlock(test_stream) as UUT: - self.assertTrue(UUT.has_child_block()) - test_stream.read(12) - self.assertTrue(UUT.has_child_block()) - test_stream.read(8) - self.assertFalse(UUT.has_child_block()) + with GSFDecoder(file_data=video_data_stream) as dec: + head = dec.file_headers - def test_contextmanager_has_child_strict_blocks(self): - """Ensure that when strict mode is enabled, has_child errors on partial blocks""" - test_stream = self._make_sample_stream(post_child_len=4) + self.assertEqual(head['created'], datetime(2018, 2, 7, 10, 38, 22)) + self.assertEqual(head['id'], UUID('163fd9b7-bef4-4d92-8488-31f3819be008')) + self.assertEqual(len(head['segments']), 1) + self.assertEqual(head['segments'][0]['id'], UUID('c6a3d3ff-74c0-446d-b59e-de1041f27e8a')) + + def test_generate_grains(self): + """Test that the generator yields each grain""" + video_data_stream = BytesIO(VIDEO_DATA) - with GSFBlock(test_stream) as UUT: - test_stream.read(12 + 8) # Read to the end of the child blocks + with GSFDecoder(file_data=video_data_stream) as dec: + grain_count = 0 + for (grain, local_id) in dec.grains(): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('49578552-fb9e-4d3e-a197-3e3c437a895d')) + self.assertEqual(grain.flow_id, UUID('6e55f251-f75a-4d56-b3af-edb8b7993c3c')) - with self.assertRaises(GSFDecodeError): - UUT.has_child_block(strict_blocks=True) + grain_count += 1 - def test_contextmanager_has_child_no_strict_blocks(self): - """Ensure that when strict mode is off, has_child doesn't error on partial blocks""" - test_stream = self._make_sample_stream(post_child_len=4) + self.assertEqual(10, grain_count) # There are 10 grains in the file - with GSFBlock(test_stream) as UUT: - test_stream.read(12 + 8) # Read to the end of the child blocks + @async_test + async def test_async_decode_headers(self): + video_data_stream = AsyncBytesIO(VIDEO_DATA) - self.assertFalse(UUT.has_child_block(strict_blocks=False)) + async with GSFDecoder(file_data=video_data_stream) as dec: + head = dec.file_headers - def test_contextmanager_child_blocks_generator(self): - """Ensure the child blocks generator returns a block, and seeks afterwards""" - test_stream = self._make_sample_stream() - with GSFBlock(test_stream) as UUT: - loop_count = 0 - child_bytes_consumed = 0 - for block in UUT.child_blocks(): - child_bytes_consumed += block.size - loop_count += 1 + self.assertEqual(head['created'], datetime(2018, 2, 7, 10, 38, 22)) + self.assertEqual(head['id'], UUID('163fd9b7-bef4-4d92-8488-31f3819be008')) + self.assertEqual(len(head['segments']), 1) + self.assertEqual(head['segments'][0]['id'], UUID('c6a3d3ff-74c0-446d-b59e-de1041f27e8a')) - # Did we get both child blocks? - self.assertEqual(2, loop_count) + @async_test + async def test_async_generate_grains(self): + """Test that the generator yields each grain""" + video_data_stream = AsyncBytesIO(VIDEO_DATA) - # Did we seek on exit from each loop iteration - self.assertEqual(child_bytes_consumed + UUT.block_start + 8, test_stream.tell()) + async with GSFDecoder(file_data=video_data_stream) as dec: + grain_count = 0 + async for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('49578552-fb9e-4d3e-a197-3e3c437a895d')) + self.assertEqual(grain.flow_id, UUID('6e55f251-f75a-4d56-b3af-edb8b7993c3c')) + grain_count += 1 -class TestGSFDecoder(TestCase): - """Tests for the GSFDecoder in its more object-oriented mode + self.assertEqual(10, grain_count) # There are 10 grains in the file - Note that very little testing of the decoded data happens here, that's handled by TestGSFLoads() - """ - def test_decode_headers(self): + @async_test + async def test_async_to_sync_generate_grains(self): + """Test that the generator yields each grain when run snchronously from asynchronous code""" + video_data_stream = BytesIO(VIDEO_DATA) + + with GSFDecoder(file_data=video_data_stream) as dec: + grain_count = 0 + for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('49578552-fb9e-4d3e-a197-3e3c437a895d')) + self.assertEqual(grain.flow_id, UUID('6e55f251-f75a-4d56-b3af-edb8b7993c3c')) + + grain_count += 1 + + self.assertEqual(10, grain_count) # There are 10 grains in the file + + @async_test + async def test_async_generate_grains_load_lazily(self): + """Test that the generator yields each grain""" + video_data_stream = AsyncBytesIO(VIDEO_DATA) + + async with GSFDecoder(file_data=video_data_stream) as dec: + grain_count = 0 + async for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('49578552-fb9e-4d3e-a197-3e3c437a895d')) + self.assertEqual(grain.flow_id, UUID('6e55f251-f75a-4d56-b3af-edb8b7993c3c')) + + self.assertIsNone(grain.data) + + await grain + + self.assertIsNotNone(grain.data) + + grain_count += 1 + + self.assertEqual(10, grain_count) # There are 10 grains in the file + + @suppress_deprecation_warnings + def test_decode_headers__deprecated(self): video_data_stream = BytesIO(VIDEO_DATA) UUT = GSFDecoder(file_data=video_data_stream) @@ -906,7 +1618,8 @@ def test_decode_headers(self): self.assertEqual(len(head['segments']), 1) self.assertEqual(head['segments'][0]['id'], UUID('c6a3d3ff-74c0-446d-b59e-de1041f27e8a')) - def test_generate_grains(self): + @suppress_deprecation_warnings + def test_generate_grains__deprecated(self): """Test that the generator yields each grain""" video_data_stream = BytesIO(VIDEO_DATA) @@ -923,13 +1636,39 @@ def test_generate_grains(self): self.assertEqual(10, grain_count) # There are 10 grains in the file + @async_test + async def test_async_comparison_of_lazy_loaded_grains(self): + async with GSFDecoder(file_data=AsyncBytesIO(VIDEO_DATA)) as dec: + grains = [grain async for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY)] + + # Restart the decoder + async with GSFDecoder(file_data=AsyncBytesIO(VIDEO_DATA)) as dec: + # Annoyingly anext isn't a global in python 3.6 + grain = (await dec.grains(loading_mode=GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE).__anext__())[0] + await grain + comp = compare_grain(grains[0], grain) + self.assertTrue(comp, msg="{!r}".format(comp)) + def test_comparison_of_lazy_loaded_grains(self): video_data_stream = BytesIO(VIDEO_DATA) + with GSFDecoder(file_data=video_data_stream) as dec: + grains = [grain for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY)] + + # Restart the decoder + video_data_stream.seek(0) + with GSFDecoder(file_data=video_data_stream) as dec: + comp = compare_grain(grains[0], next(dec.grains(loading_mode=GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE))[0]) + self.assertTrue(comp, msg="{!s}".format(comp)) + + @suppress_deprecation_warnings + def test_comparison_of_lazy_loaded_grains__deprecated(self): + video_data_stream = BytesIO(VIDEO_DATA) + UUT = GSFDecoder(file_data=video_data_stream) UUT.decode_file_headers() - grains = [grain for (grain, local_id) in UUT.grains(load_lazily=False)] + grains = [grain for (grain, local_id) in UUT.grains(loading_mode=GrainDataLoadingMode.LOAD_IMMEDIATELY)] # Restart the decoder video_data_stream.seek(0) @@ -938,9 +1677,69 @@ def test_comparison_of_lazy_loaded_grains(self): self.assertTrue(compare_grain(grains[0], next(UUT.grains(load_lazily=True))[0])) + @async_test + async def test_async_local_id_filtering(self): + interleaved_data_stream = AsyncBytesIO(INTERLEAVED_DATA) + + async with GSFDecoder(file_data=interleaved_data_stream) as dec: + local_ids = set() + flow_ids = set() + async for (grain, local_id) in dec.grains(): + local_ids.add(local_id) + flow_ids.add(grain.flow_id) + + self.assertEqual(local_ids, set([1, 2])) + self.assertEqual(flow_ids, set([UUID('28e4e09e-3517-11e9-8da2-5065f34ed007'), + UUID('2472f38e-3517-11e9-8da2-5065f34ed007')])) + + async with GSFDecoder(file_data=interleaved_data_stream) as dec: + async for (grain, local_id) in dec.grains(local_ids=[1]): + self.assertIsInstance(grain, AUDIOGRAIN) + self.assertEqual(grain.source_id, UUID('1f8fd27e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(grain.flow_id, UUID('28e4e09e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(local_id, 1) + + async with GSFDecoder(file_data=interleaved_data_stream) as dec: + async for (grain, local_id) in dec.grains(local_ids=[2]): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('1f8fd27e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(grain.flow_id, UUID('2472f38e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(local_id, 2) + def test_local_id_filtering(self): interleaved_data_stream = BytesIO(INTERLEAVED_DATA) + with GSFDecoder(file_data=interleaved_data_stream) as dec: + local_ids = set() + flow_ids = set() + for (grain, local_id) in dec.grains(): + local_ids.add(local_id) + flow_ids.add(grain.flow_id) + + self.assertEqual(local_ids, set([1, 2])) + self.assertEqual(flow_ids, set([UUID('28e4e09e-3517-11e9-8da2-5065f34ed007'), + UUID('2472f38e-3517-11e9-8da2-5065f34ed007')])) + + interleaved_data_stream.seek(0) + with GSFDecoder(file_data=interleaved_data_stream) as dec: + for (grain, local_id) in dec.grains(local_ids=[1]): + self.assertIsInstance(grain, AUDIOGRAIN) + self.assertEqual(grain.source_id, UUID('1f8fd27e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(grain.flow_id, UUID('28e4e09e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(local_id, 1) + + interleaved_data_stream.seek(0) + with GSFDecoder(file_data=interleaved_data_stream) as dec: + for (grain, local_id) in dec.grains(local_ids=[2]): + self.assertIsInstance(grain, VIDEOGRAIN) + self.assertEqual(grain.source_id, UUID('1f8fd27e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(grain.flow_id, UUID('2472f38e-3517-11e9-8da2-5065f34ed007')) + self.assertEqual(local_id, 2) + + @suppress_deprecation_warnings + def test_local_id_filtering__deprecated(self): + interleaved_data_stream = BytesIO(INTERLEAVED_DATA) + UUT = GSFDecoder(file_data=interleaved_data_stream) UUT.decode_file_headers() @@ -972,7 +1771,8 @@ def test_local_id_filtering(self): self.assertEqual(grain.flow_id, UUID('2472f38e-3517-11e9-8da2-5065f34ed007')) self.assertEqual(local_id, 2) - def test_skip_grain_data(self): + @suppress_deprecation_warnings + def test_skip_grain_data__deprecated(self): """Test that the `skip_data` parameter causes grain data to be seeked over""" grain_size = 194612 # Parsed from examples/video.gsf hex dump grdt_block_size = 194408 # Parsed from examples/video.gsf hex dump @@ -1009,6 +1809,51 @@ def test_lazy_load_grain_data(self): video_data_stream = BytesIO(VIDEO_DATA) + reader_mock = mock.MagicMock(side_effect=video_data_stream.read) + with mock.patch.object(video_data_stream, "read", new=reader_mock): + grains = [] + with GSFDecoder(file_data=video_data_stream) as dec: + reader_mock.reset_mock() + + for (grain, local_id) in dec.grains(loading_mode=GrainDataLoadingMode.ALWAYS_DEFER_LOAD_IF_POSSIBLE): + grains.append(grain) + # Add up the bytes read for this grain, then reset the read counter + bytes_read = 0 + for args, _ in reader_mock.call_args_list: + bytes_read += args[0] + + reader_mock.reset_mock() + + # No more than the number of bytes in the header should have been read + # However some header bytes may be seeked over instead + self.assertGreater(bytes_read, 0) + self.assertLessEqual(bytes_read, grain_header_size) + + for grain in grains: + reader_mock.reset_mock() + + self.assertEqual(grain.length, grain_data_size) + reader_mock.assert_not_called() + + x = grain.data[grain_data_size-1] # noqa: F841 + bytes_read = 0 + for (args, _) in reader_mock.call_args_list: + bytes_read += args[0] + + self.assertEqual(bytes_read, grain_data_size) + self.assertEqual(grain.length, grain_data_size) + + @suppress_deprecation_warnings + def test_lazy_load_grain_data__deprecated(self): + """Test that the `load_lazily` parameter causes grain data to be seeked over, + but then loaded invisibly when needed later""" + grain_size = 194612 # Parsed from examples/video.gsf hex dump + grdt_block_size = 194408 # Parsed from examples/video.gsf hex dump + grain_header_size = grain_size - grdt_block_size + grain_data_size = grdt_block_size - 8 + + video_data_stream = BytesIO(VIDEO_DATA) + UUT = GSFDecoder(file_data=video_data_stream) UUT.decode_file_headers() @@ -1135,6 +1980,37 @@ def test_load_video(self): self.assertEqual(len(grain.data), grain.components[0].length + grain.components[1].length + grain.components[2].length) + def test_load_uses_custom_grain_function(self): + file = BytesIO(VIDEO_DATA) + grain_parser = mock.MagicMock(name="grain_parser") + (head, segments) = load(file, parse_grain=grain_parser) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), 10) + self.assertEqual(grain_parser.call_count, 10) + + @async_test + async def test_async_load_uses_custom_grain_function(self): + file = AsyncBytesIO(VIDEO_DATA) + grain_parser = mock.MagicMock(name="grain_parser") + (head, segments) = await load(file, parse_grain=grain_parser) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), 10) + self.assertEqual(grain_parser.call_count, 10) + + def test_loads_uses_custom_grain_function(self): + s = VIDEO_DATA + grain_parser = mock.MagicMock(name="grain_parser") + (head, segments) = loads(s, parse_grain=grain_parser) + + self.assertEqual(len(segments), 1) + self.assertIn(1, segments) + self.assertEqual(len(segments[1]), 10) + self.assertEqual(grain_parser.call_count, 10) + def test_loads_audio(self): (head, segments) = loads(AUDIO_DATA) @@ -1342,6 +2218,50 @@ def test_loads_decodes_tils(self): 'frame_rate_denominator': 1, 'drop_frame': False}}]) + @async_test + async def test_async_load_decodes_tils(self): + src_id = UUID('c707d64c-1596-11e8-a3fb-dca904824eec') + flow_id = UUID('da78668a-1596-11e8-a577-dca904824eec') + fp = AsyncBytesIO(b"SSBBgrsg\x07\x00\x00\x00" + + (b"head\x41\x00\x00\x00" + + b"\xd1\x9c\x0b\x91\x15\x90\x11\xe8\x85\x80\xdc\xa9\x04\x82N\xec" + + b"\xbf\x07\x03\x1d\x0f\x0f\x0f" + + (b"segm\x22\x00\x00\x00" + + b"\x01\x00" + + b"\xd3\xe1\x91\xf0\x15\x94\x11\xe8\x91\xac\xdc\xa9\x04\x82N\xec" + + b"\x01\x00\x00\x00\x00\x00\x00\x00")) + + (b"grai\x8d\x00\x00\x00" + + b"\x01\x00" + + (b"gbhd\x83\x00\x00\x00" + + src_id.bytes + + flow_id.bytes + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + b"\x00\x00\x00\x00\x00\x00\x00\x00" + + b"\x00\x00\x00\x00\x00\x00\x00\x00" + + (b"tils\x27\x00\x00\x00" + + b"\x01\x00" + + b"dummy timecode\x00\x00" + + b"\x07\x00\x00\x00" + + b"\x19\x00\x00\x00\x01\x00\x00\x00" + + b"\x00"))) + + (b"grai\x08\x00\x00\x00")) + (head, segments) = await load(fp) + + self.assertEqual(head['id'], UUID('d19c0b91-1590-11e8-8580-dca904824eec')) + self.assertEqual(head['created'], datetime(1983, 3, 29, 15, 15, 15)) + self.assertEqual(len(head['segments']), 1) + self.assertEqual(head['segments'][0]['local_id'], 1) + self.assertEqual(head['segments'][0]['id'], UUID('d3e191f0-1594-11e8-91ac-dca904824eec')) + self.assertEqual(head['segments'][0]['tags'], []) + self.assertEqual(head['segments'][0]['count'], 1) + self.assertEqual(head['tags'], []) + self.assertEqual(segments[1][0].timelabels, [{'tag': 'dummy timecode', 'timelabel': {'frames_since_midnight': 7, + 'frame_rate_numerator': 25, + 'frame_rate_denominator': 1, + 'drop_frame': False}}]) + def test_loads_raises_when_grain_type_unknown(self): with self.assertRaises(GSFDecodeError) as cm: src_id = UUID('c707d64c-1596-11e8-a3fb-dca904824eec') diff --git a/tests/test_iobytes.py b/tests/test_iobytes.py index af5938d..b7ea622 100644 --- a/tests/test_iobytes.py +++ b/tests/test_iobytes.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2019 British Broadcasting Corporation # @@ -15,8 +14,6 @@ # limitations under the License. # -from __future__ import print_function - from unittest import TestCase import mock @@ -25,10 +22,10 @@ from hypothesis import given from hypothesis.strategies import integers -from six import int2byte, BytesIO, binary_type +from io import BytesIO -TEST_DATA = b''.join(int2byte((x//256) % 256) + int2byte(x % 256) for x in range(0, 65536)) +TEST_DATA = b''.join(bytes(((x//256) % 256, (x % 256))) for x in range(0, 65536)) class IncorrectAccess (Exception): @@ -42,7 +39,7 @@ def test_read(self, start, length): iostream = BytesIO(TEST_DATA) orig_loc = iostream.tell() iobytes = IOBytes(iostream, start, length) - data = binary_type(iobytes) + data = bytes(iobytes) self.assertEqual(len(data), length) self.assertEqual(data, TEST_DATA[start:start + length]) self.assertEqual(orig_loc, iostream.tell()) diff --git a/tests/test36_numpy_videograin.py b/tests/test_numpy_videograin.py similarity index 91% rename from tests/test36_numpy_videograin.py rename to tests/test_numpy_videograin.py index 5157437..59158e4 100644 --- a/tests/test36_numpy_videograin.py +++ b/tests/test_numpy_videograin.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -19,32 +18,28 @@ import uuid from mediagrains.numpy import VideoGrain, VIDEOGRAIN -from mediagrains_py36.numpy.videograin import _dtype_from_cogframeformat +from mediagrains.numpy.videograin import _dtype_from_cogframeformat from mediagrains.cogenums import ( CogFrameFormat, CogFrameLayout, COG_FRAME_FORMAT_BYTES_PER_VALUE, COG_FRAME_FORMAT_H_SHIFT, COG_FRAME_FORMAT_V_SHIFT, - COG_FRAME_IS_PACKED, - COG_FRAME_IS_COMPRESSED, COG_FRAME_IS_PLANAR, COG_FRAME_IS_PLANAR_RGB, - COG_FRAME_FORMAT_ACTIVE_BITS, - COG_PLANAR_FORMAT, - PlanarChromaFormat) + COG_FRAME_FORMAT_ACTIVE_BITS) from mediatimestamp.immutable import Timestamp, TimeRange import mock from fractions import Fraction from copy import copy, deepcopy from typing import Tuple, Optional +from fixtures import async_test + from itertools import chain, repeat import numpy as np -from pdb import set_trace - class TestGrain (TestCase): def _get_bitdepth(self, fmt): @@ -364,9 +359,9 @@ def _test_pattern_yuv(self, fmt: CogFrameFormat) -> Tuple[np.ndarray, np.ndarray bd = self._get_bitdepth(fmt) (hs, vs, _) = self._get_hs_vs_and_bps(fmt) - Y = (R*0.2126 + G*0.7152 + B*0.0722) + Y = (R*0.2126 + G*0.7152 + B*0.0722) U = (R*-0.114572 - G*0.385428 + B*0.5 + (1 << (bd - 1))) - V = (R*0.5 - G*0.454153 - B*0.045847 + (1 << (bd - 1))) + V = (R*0.5 - G*0.454153 - B*0.045847 + (1 << (bd - 1))) if hs == 1: U = (U[0::2, :] + U[1::2, :])/2 @@ -375,7 +370,9 @@ def _test_pattern_yuv(self, fmt: CogFrameFormat) -> Tuple[np.ndarray, np.ndarray U = (U[:, 0::2] + U[:, 1::2])/2 V = (V[:, 0::2] + V[:, 1::2])/2 - return (np.around(Y).astype(_dtype_from_cogframeformat(fmt)), np.around(U).astype(_dtype_from_cogframeformat(fmt)), np.around(V).astype(_dtype_from_cogframeformat(fmt))) + return (np.around(Y).astype(_dtype_from_cogframeformat(fmt)), + np.around(U).astype(_dtype_from_cogframeformat(fmt)), + np.around(V).astype(_dtype_from_cogframeformat(fmt))) def _test_pattern_v210(self) -> np.ndarray: (Y, U, V) = self._test_pattern_yuv(CogFrameFormat.S16_422_10BIT) @@ -388,14 +385,13 @@ def _test_pattern_v210(self) -> np.ndarray: vv = chain(iter(V[:, y]), repeat(0)) for x in range(0, 8): - output[y*32 + 4*x + 0] = next(uu) | (next(yy) << 10) | (next(vv) << 20) - output[y*32 + 4*x + 1] = next(yy) | (next(uu) << 10) | (next(yy) << 20) - output[y*32 + 4*x + 2] = next(vv) | (next(yy) << 10) | (next(uu) << 20) - output[y*32 + 4*x + 3] = next(yy) | (next(vv) << 10) | (next(yy) << 20) + output[y*32 + 4*x + 0] = next(uu) | (next(yy) << 10) | (next(vv) << 20) + output[y*32 + 4*x + 1] = next(yy) | (next(uu) << 10) | (next(yy) << 20) + output[y*32 + 4*x + 2] = next(vv) | (next(yy) << 10) | (next(uu) << 20) + output[y*32 + 4*x + 3] = next(yy) | (next(vv) << 10) | (next(yy) << 20) return output - def write_test_pattern(self, grain): fmt = grain.format @@ -491,33 +487,67 @@ def test_video_grain_create(self): if fmt is not CogFrameFormat.v210: self.assertComponentsAreModifiable(grain) - def test_video_grain_convert(self): + @async_test + async def test_video_grain_async_create(self): src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") cts = Timestamp.from_tai_sec_nsec("417798915:0") ots = Timestamp.from_tai_sec_nsec("417798915:5") sts = Timestamp.from_tai_sec_nsec("417798915:10") + async def _get_data(): + _data = bytearray(16*16*3) + for i in range(0, 3): + for y in range(0, 16): + for x in range(0, 16): + _data[(i*16 + y)*16 + x] = x + (y << 4) + return _data + + data_awaitable = _get_data() + + with mock.patch.object(Timestamp, "get_time", return_value=cts): + grain = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, + cog_frame_format=CogFrameFormat.U8_444, + width=16, height=16, cog_frame_layout=CogFrameLayout.FULL_FRAME, + data=data_awaitable) + + self.assertIsNone(grain.data) + self.assertEqual(len(grain.component_data), 0) + + async with grain as _grain: + for y in range(0, 16): + for x in range(0, 16): + self.assertEqual(_grain.component_data.Y[x, y], x + (y << 4)) + self.assertEqual(_grain.component_data.U[x, y], x + (y << 4)) + self.assertEqual(_grain.component_data.V[x, y], x + (y << 4)) + + def test_video_grain_convert(self): + src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") + flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") + cts = Timestamp.from_tai_sec_nsec("417798915:0") + ots = Timestamp.from_tai_sec_nsec("417798915:5") + sts = Timestamp.from_tai_sec_nsec("417798915:10") def pairs_from(fmts): for fmt_in in fmts: for fmt_out in fmts: yield (fmt_in, fmt_out) - fmts = [CogFrameFormat.YUYV, CogFrameFormat.UYVY, CogFrameFormat.U8_444, CogFrameFormat.U8_422, CogFrameFormat.U8_420, # All YUV 8bit formats - CogFrameFormat.RGB, CogFrameFormat.U8_444_RGB, CogFrameFormat.RGBx, CogFrameFormat.xRGB, CogFrameFormat.BGRx, CogFrameFormat.xBGR, # All 8-bit 3 component RGB formats - CogFrameFormat.v216, CogFrameFormat.S16_444, CogFrameFormat.S16_422, CogFrameFormat.S16_420, # All YUV 16bit formats - CogFrameFormat.S16_444_10BIT, CogFrameFormat.S16_422_10BIT, CogFrameFormat.S16_420_10BIT, # All YUV 10bit formats except for v210 - CogFrameFormat.v210, # v210, may the gods be merciful to us for including it - CogFrameFormat.S16_444_12BIT, CogFrameFormat.S16_422_12BIT, CogFrameFormat.S16_420_12BIT, # All YUV 12bit formats - CogFrameFormat.S32_444, CogFrameFormat.S32_422, CogFrameFormat.S32_420, # All YUV 32bit formats - CogFrameFormat.S16_444_RGB, CogFrameFormat.S16_444_10BIT_RGB, CogFrameFormat.S16_444_12BIT_RGB, CogFrameFormat.S32_444_RGB] # Other planar RGB formats + fmts = [CogFrameFormat.YUYV, CogFrameFormat.UYVY, CogFrameFormat.U8_444, CogFrameFormat.U8_422, CogFrameFormat.U8_420, # All YUV 8bit formats + CogFrameFormat.RGB, CogFrameFormat.U8_444_RGB, CogFrameFormat.RGBx, CogFrameFormat.xRGB, CogFrameFormat.BGRx, CogFrameFormat.xBGR, + # All 8-bit 3 component RGB formats + CogFrameFormat.v216, CogFrameFormat.S16_444, CogFrameFormat.S16_422, CogFrameFormat.S16_420, # All YUV 16bit formats + CogFrameFormat.S16_444_10BIT, CogFrameFormat.S16_422_10BIT, CogFrameFormat.S16_420_10BIT, # All YUV 10bit formats except for v210 + CogFrameFormat.v210, # v210, may the gods be merciful to us for including it + CogFrameFormat.S16_444_12BIT, CogFrameFormat.S16_422_12BIT, CogFrameFormat.S16_420_12BIT, # All YUV 12bit formats + CogFrameFormat.S32_444, CogFrameFormat.S32_422, CogFrameFormat.S32_420, # All YUV 32bit formats + CogFrameFormat.S16_444_RGB, CogFrameFormat.S16_444_10BIT_RGB, CogFrameFormat.S16_444_12BIT_RGB, CogFrameFormat.S32_444_RGB] # Other planar RGB for (fmt_in, fmt_out) in pairs_from(fmts): with self.subTest(fmt_in=fmt_in, fmt_out=fmt_out): with mock.patch.object(Timestamp, "get_time", return_value=cts): grain_in = VideoGrain(src_id, flow_id, origin_timestamp=ots, sync_timestamp=sts, - cog_frame_format=fmt_in, - width=16, height=16, cog_frame_layout=CogFrameLayout.FULL_FRAME) + cog_frame_format=fmt_in, + width=16, height=16, cog_frame_layout=CogFrameLayout.FULL_FRAME) self.assertIsVideoGrain(fmt_in, width=16, height=16)(grain_in) self.write_test_pattern(grain_in) @@ -542,13 +572,13 @@ def pairs_from(fmts): # If we've increased bit-depth there will be rounding errors if self._get_bitdepth(fmt_out) > self._get_bitdepth(fmt_in): - self.assertMatchesTestPattern(grain_out, max_diff=1 << (self._get_bitdepth(fmt_out) + 2 - self._get_bitdepth(fmt_in))) + self.assertMatchesTestPattern(grain_out, max_diff=1 << (self._get_bitdepth(fmt_out) + 2 - self._get_bitdepth(fmt_in))) # If we're changing from yuv to rgb then there's some potential for floating point errors, depending on the sizes elif self._get_bitdepth(fmt_in) >= 16 and not self._is_rgb(fmt_in) and fmt_out == CogFrameFormat.S16_444_RGB: self.assertMatchesTestPattern(grain_out, max_diff=2) elif self._get_bitdepth(fmt_in) == 32 and not self._is_rgb(fmt_in) and fmt_out == CogFrameFormat.S32_444_RGB: - self.assertMatchesTestPattern(grain_out, max_diff=1 << 10) # The potential errors in 32 bit conversions are very large + self.assertMatchesTestPattern(grain_out, max_diff=1 << 10) # The potential errors in 32 bit conversions are very large # If we've decreased bit-depth *and* or changed from rgb to yuv then there is a smaller scope for error elif ((self._get_bitdepth(fmt_out) < self._get_bitdepth(fmt_in)) or @@ -581,7 +611,6 @@ def pairs_from(fmts): elif self._is_rgb(fmt_in): self.assertMatchesTestPattern(grain_rev, max_diff=4) - def test_video_grain_create_discontiguous(self): src_id = uuid.UUID("f18ee944-0841-11e8-b0b0-17cef04bd429") flow_id = uuid.UUID("f79ce4da-0841-11e8-9a5b-dfedb11bafeb") diff --git a/tests/test_psnr.py b/tests/test_psnr.py index 56f7981..d1b5691 100644 --- a/tests/test_psnr.py +++ b/tests/test_psnr.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2019 British Broadcasting Corporation # @@ -15,9 +14,6 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - from unittest import TestCase from sys import version_info import uuid diff --git a/tests/test_testsignalgenerator.py b/tests/test_testsignalgenerator.py index 6547fcb..80b7200 100644 --- a/tests/test_testsignalgenerator.py +++ b/tests/test_testsignalgenerator.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright 2018 British Broadcasting Corporation # @@ -15,15 +14,11 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import absolute_import - from unittest import TestCase from uuid import UUID from mediatimestamp.immutable import Timestamp, TimeOffset from fractions import Fraction -from six import next import struct from math import sin, pi @@ -643,7 +638,6 @@ def test_movingbar_colourbars75_s16_422_10bit(self): self.assertEqual(V[y*grain.components[2].stride + 2*x + 0], expected[x//(width//16)][2] & 0xFF) self.assertEqual(V[y*grain.components[2].stride + 2*x + 1], expected[x//(width//16)][2] >> 8) - ts = Timestamp.from_count(ts.to_count(25, 1) + 1, 25, 1) diff --git a/tox.ini b/tox.ini index 46f3906..0e58239 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,15 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py36 +envlist = py36 [testenv] commands = - py27: python -m unittest discover -s tests -p test_*.py - py35: python -m unittest discover -s tests -p test_*.py - py36: python -m unittest discover -s tests -p test*_*.py - py37: python -m unittest discover -s tests -p test*_*.py + python -m unittest discover -s tests -p test_*.py deps = hypothesis >= 4.0.0 mock coverage - py36: aiofiles + aiofiles + mypy + flake8