-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge dev into master post v2.7.0
- Loading branch information
Showing
54 changed files
with
4,804 additions
and
1,981 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,9 @@ | ||
| @Library("[email protected]") _ | ||
|
|
||
| /* | ||
| Runs the following steps in parallel and reports results to GitHub: | ||
| Runs the following steps in series and reports results to GitHub: | ||
| - Lint using flake8 | ||
| - Run Python 2.7 unit tests in tox | ||
| - Type check using mypy | ||
| - Run Python 3 unit tests in tox | ||
| - Build Debian packages for supported Ubuntu versions | ||
|
|
@@ -44,84 +44,62 @@ pipeline { | |
| } | ||
| stage ("Tests") { | ||
| stages { | ||
| stage ("Py2.7 Linting Check") { | ||
| stage ("Py36 Linting Check") { | ||
| steps { | ||
| script { | ||
| env.lint27_result = "FAILURE" | ||
| env.lint3_result = "FAILURE" | ||
| } | ||
| bbcGithubNotify(context: "lint/flake8_27", status: "PENDING") | ||
| bbcGithubNotify(context: "lint/flake8_3", status: "PENDING") | ||
| // Run the linter | ||
| sh 'python2.7 -m flake8 --filename=mediagrains/*.py,tests/test_*.py' | ||
| sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-lint make lint' | ||
| script { | ||
| env.lint27_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| } | ||
| } | ||
| post { | ||
| always { | ||
| bbcGithubNotify(context: "lint/flake8_27", status: env.lint27_result) | ||
| bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result) | ||
| } | ||
| } | ||
| } | ||
| stage ("Py36 Linting Check") { | ||
| stage ("Py36 Type Check") { | ||
| steps { | ||
| script { | ||
| env.lint3_result = "FAILURE" | ||
| env.mypy_result = "FAILURE" | ||
| } | ||
| bbcGithubNotify(context: "lint/flake8_3", status: "PENDING") | ||
| bbcGithubNotify(context: "type/mypy", status: "PENDING") | ||
| // Run the linter | ||
| sh 'python3 -m flake8 --filename=mediagrains/*.py,mediagrains_async/*.py,tests/test_*.py,tests/atest_*.py' | ||
| sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-mypy make mypy' | ||
| script { | ||
| env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| env.mypy_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| } | ||
| } | ||
| post { | ||
| always { | ||
| bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result) | ||
| bbcGithubNotify(context: "type/mypy", status: env.mypy_result) | ||
| } | ||
| } | ||
| } | ||
| stage ("Build Docs") { | ||
| steps { | ||
| sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs' | ||
| } | ||
| steps { | ||
| sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs' | ||
| } | ||
| } | ||
| stage ("Unit Tests") { | ||
| stages { | ||
| stage ("Python 2.7 Unit Tests") { | ||
| steps { | ||
| script { | ||
| env.py27_result = "FAILURE" | ||
| } | ||
| bbcGithubNotify(context: "tests/py27", status: "PENDING") | ||
| // Use a workdirectory in /tmp to avoid shebang length limitation | ||
| sh 'tox -e py27 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py27' | ||
| script { | ||
| env.py27_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| } | ||
| } | ||
| post { | ||
| always { | ||
| bbcGithubNotify(context: "tests/py27", status: env.py27_result) | ||
| } | ||
| } | ||
| stage ("Python 3 Unit Tests") { | ||
| steps { | ||
| script { | ||
| env.py36_result = "FAILURE" | ||
| } | ||
| stage ("Python 3 Unit Tests") { | ||
| steps { | ||
| script { | ||
| env.py36_result = "FAILURE" | ||
| } | ||
| bbcGithubNotify(context: "tests/py36", status: "PENDING") | ||
| // Use a workdirectory in /tmp to avoid shebang length limitation | ||
| sh 'tox -e py36 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py36' | ||
| script { | ||
| env.py36_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| } | ||
| } | ||
| post { | ||
| always { | ||
| bbcGithubNotify(context: "tests/py36", status: env.py36_result) | ||
| } | ||
| } | ||
| bbcGithubNotify(context: "tests/py36", status: "PENDING") | ||
| // Use a workdirectory in /tmp to avoid shebang length limitation | ||
| sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-py36 make test' | ||
| script { | ||
| env.py36_result = "SUCCESS" // This will only run if the sh above succeeded | ||
| } | ||
| } | ||
| post { | ||
| always { | ||
| bbcGithubNotify(context: "tests/py36", status: env.py36_result) | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -218,7 +196,6 @@ pipeline { | |
| } | ||
| bbcGithubNotify(context: "pypi/upload", status: "PENDING") | ||
| sh 'rm -rf dist/*' | ||
| bbcMakeGlobalWheel("py27") | ||
| bbcMakeGlobalWheel("py36") | ||
| bbcTwineUpload(toxenv: "py36", pypi: true) | ||
| script { | ||
|
|
@@ -246,7 +223,6 @@ pipeline { | |
| } | ||
| bbcGithubNotify(context: "artifactory/upload", status: "PENDING") | ||
| sh 'rm -rf dist/*' | ||
| bbcMakeGlobalWheel("py27") | ||
| bbcMakeGlobalWheel("py36") | ||
| bbcTwineUpload(toxenv: "py36", pypi: false) | ||
| script { | ||
|
|
@@ -276,9 +252,9 @@ pipeline { | |
| script { | ||
| for (def dist in bbcGetSupportedUbuntuVersions()) { | ||
| bbcDebUpload(sourceFiles: "_result/${dist}-amd64/*", | ||
| removePrefix: "_result/${dist}-amd64", | ||
| dist: "${dist}", | ||
| apt_repo: "ap/python") | ||
| removePrefix: "_result/${dist}-amd64", | ||
| dist: "${dist}", | ||
| apt_repo: "ap/python") | ||
| } | ||
| } | ||
| script { | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,11 +10,14 @@ that nicely wrap those grains, as well as a full serialisation and | |
| deserialisation library for GSF format. Please read the pydoc | ||
| documentation for more details. | ||
|
|
||
| Some useful tools for handling the Grain Sequence Format (GSF) file format | ||
| are also included - see [Tools](#tools). | ||
|
|
||
| ## Installation | ||
|
|
||
| ### Requirements | ||
|
|
||
| * A working Python 2.7 or Python 3.6+ installation | ||
| * A working Python 3.6+ installation | ||
| * BBC R&D's internal deb repository set up as a source for apt (if installing via apt-get) | ||
| * The tool [tox](https://tox.readthedocs.io/en/latest/) is needed to run the unittests, but not required to use the library. | ||
|
|
||
|
|
@@ -25,7 +28,7 @@ documentation for more details. | |
| $ pip install mediagrains | ||
|
|
||
| # Install via apt-get | ||
| $ apt-get install python-mediagrains python3-mediagrains | ||
| $ apt-get install python3-mediagrains | ||
|
|
||
| # Install directly from source repo | ||
| $ git clone [email protected]:bbc/rd-apmm-python-lib-mediagrains.git | ||
|
|
@@ -85,7 +88,7 @@ it with colour-bars: | |
| ... i += 1 | ||
| ``` | ||
|
|
||
| (In python3.6+ a more natural interface for accessing data exists in the form of numpy arrays. See later.) | ||
| (a more natural interface for accessing data exists in the form of numpy arrays. See later.) | ||
|
|
||
| The object grain can then be freely used for whatever video processing | ||
| is desired, or it can be serialised into a GSF file as follows: | ||
|
|
@@ -161,9 +164,9 @@ between two grains, both as a printed string (as seen above) and also | |
| in a data-centric fashion as a tree structure which can be | ||
| interrogated in code. | ||
|
|
||
| ### Numpy arrays (Python 3.6+) | ||
| ### Numpy arrays | ||
|
|
||
| In python 3.6 or higher an additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily: | ||
| An additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily: | ||
|
|
||
| ```Python console | ||
| >>> from mediagrains.numpy import VideoGrain | ||
|
|
@@ -193,6 +196,30 @@ The API is well documented in the docstrings of the module mediagrains, to view: | |
| pydoc mediagrains | ||
| ``` | ||
|
|
||
| ## Tools | ||
| Some tools are installed with the library to make working with the Grain Sequence Format (GSF) file format easier. | ||
|
|
||
| * `wrap_video_in_gsf` - Provides a means to read raw video essence and generate a GSF file. | ||
| * `wrap_audio_in_gsf` - As above, but for audio. | ||
| * `extract_from_gsf` - Read a GSF file and dump out the raw essence within. | ||
| * `gsf_probe` - Read metadata about the segments in a GSF file. | ||
|
|
||
| For example, to generate a GSF file containing a test pattern from `ffmpeg`, dump the metadata and then play it out | ||
| again: | ||
| ```bash | ||
| ffmpeg -f lavfi -i testsrc=duration=20:size=1920x1080:rate=25 -pix_fmt yuv422p10le -c:v rawvideo -f rawvideo - | \ | ||
| wrap_video_in_gsf - output.gsf --size 1920x1080 --format S16_422_10BIT --rate 25 | ||
| gsf_probe output.gsf | ||
| extract_gsf_essence output.gsf - | ffplay -f rawvideo -pixel_format yuv422p10 -video_size 1920x1080 -framerate 25 pipe:0 | ||
| ``` | ||
|
|
||
| To do the same with a sine wave: | ||
| ```bash | ||
| ffmpeg -f lavfi -i "sine=frequency=1000:duration=5" -f s16le -ac 2 - | wrap_audio_in_gsf - output_audio.gsf --sample-rate 44100 | ||
| gsf_probe output_audio.gsf | ||
| extract_gsf_essence output_audio.gsf - | ffplay -f s16le -ac 2 -ar 44100 pipe:0 | ||
| ``` | ||
|
|
||
| ## Development | ||
| ### Testing | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.