From 4e6fc6e30fff79dca9eea17d2a9f268e9ef5b764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20Zaczy=C5=84ski?= Date: Thu, 28 Mar 2024 18:43:58 +0100 Subject: [PATCH] __pycache__: Initial commit (materials) --- python-pycache/Calculator.java | 9 +++ python-pycache/README.md | 81 +++++++++++++++++++ .../example-1/project/calculator.py | 1 + .../example-1/project/mathematics/__init__.py | 0 .../mathematics/arithmetic/__init__.py | 0 .../project/mathematics/arithmetic/add.py | 2 + .../project/mathematics/arithmetic/sub.py | 2 + .../project/mathematics/geometry/__init__.py | 0 .../project/mathematics/geometry/shapes.py | 0 python-pycache/example-2/arithmetic.py | 2 + python-pycache/example-2/calculator.py | 3 + .../example-3/project/arithmetic/__init__.py | 0 .../example-3/project/arithmetic/add.py | 2 + .../example-3/project/arithmetic/sub.py | 2 + .../example-3/project/calculator.py | 3 + python-pycache/pyclean.ps1 | 1 + python-pycache/pyclean.sh | 3 + python-pycache/xray.py | 51 ++++++++++++ 18 files changed, 162 insertions(+) create mode 100644 python-pycache/Calculator.java create mode 100644 python-pycache/README.md create mode 100644 python-pycache/example-1/project/calculator.py create mode 100644 python-pycache/example-1/project/mathematics/__init__.py create mode 100644 python-pycache/example-1/project/mathematics/arithmetic/__init__.py create mode 100644 python-pycache/example-1/project/mathematics/arithmetic/add.py create mode 100644 python-pycache/example-1/project/mathematics/arithmetic/sub.py create mode 100644 python-pycache/example-1/project/mathematics/geometry/__init__.py create mode 100644 python-pycache/example-1/project/mathematics/geometry/shapes.py create mode 100644 python-pycache/example-2/arithmetic.py create mode 100644 python-pycache/example-2/calculator.py create mode 100644 python-pycache/example-3/project/arithmetic/__init__.py create mode 100644 python-pycache/example-3/project/arithmetic/add.py create mode 100644 python-pycache/example-3/project/arithmetic/sub.py create mode 100644 python-pycache/example-3/project/calculator.py create mode 100755 python-pycache/pyclean.ps1 create mode 100755 python-pycache/pyclean.sh create mode 100644 python-pycache/xray.py diff --git a/python-pycache/Calculator.java b/python-pycache/Calculator.java new file mode 100644 index 0000000000..f033e2ddef --- /dev/null +++ b/python-pycache/Calculator.java @@ -0,0 +1,9 @@ +public class Calculator { + public static void main(String[] args) { + add(3, 4); + } + + private static int add(int a, int b) { + return a + b; + } +} diff --git a/python-pycache/README.md b/python-pycache/README.md new file mode 100644 index 0000000000..eb45d5d5d7 --- /dev/null +++ b/python-pycache/README.md @@ -0,0 +1,81 @@ +# What Is the __pycache__ Folder in Python? + +Source code, shell scripts, and example projects for the [What Is the `__pycache__` Folder in Python?](https://realpython.com/python-pycache/) tutorial on Real Python. + +## Setup + +You don't need to create a virtual environments because you won't be installing anything. + +## Cleanup Scripts + +To recursively remove the `__pycache__` folders on macOS and Linux: + +```shell +$ ./pyclean.sh +``` + +To do the same on Windows in PowerShell: + +```shell +PS> .\pyclean.ps1 +``` + +## X-Ray of `.pyc` Files + +Compile sample bytecode into timestamp-based `.pyc` files: + +```shell +$ ./pyclean.sh +$ python -m compileall --invalidation-mode timestamp example-2/ +$ python xray.py example-2/__pycache__/arithmetic*.pyc +{'magic_number': b'\xcb\r\r\n', + 'magic_int': 3531, + 'python_version': '3.12', + 'bit_field': 0, + 'pyc_type': , + 'timestamp': datetime.datetime(2024, 3, 28, 17, 8, 22, tzinfo=datetime.timezone.utc), + 'file_size': 32} +``` + +Compile sample bytecode into unchecked hash-based `.pyc` files: + +```shell +$ ./pyclean.sh +$ python -m compileall --invalidation-mode unchecked-hash example-2/ +$ python xray.py example-2/__pycache__/arithmetic*.pyc +{'magic_number': b'\xcb\r\r\n', + 'magic_int': 3531, + 'python_version': '3.12', + 'bit_field': 1, + 'pyc_type': , + 'hash_value': b'\xf3\xdd\x87j\x8d>\x0e)'} +``` + +Compile sample bytecode into checked hash-based `.pyc` files: + +```shell +$ ./pyclean.sh +$ python -m compileall --invalidation-mode checked-hash example-2/ +$ python xray.py example-2/__pycache__/arithmetic*.pyc +{'magic_number': b'\xcb\r\r\n', + 'magic_int': 3531, + 'python_version': '3.12', + 'bit_field': 3, + 'pyc_type': , + 'hash_value': b'\xf3\xdd\x87j\x8d>\x0e)'} +``` + +## Java Bytecode Compiler + +Compile the source code upfront and run the resulting class file: + +```shell +$ javac Calculator.java +$ time java Calculator +``` + +Let the `java` command handle the compilation: + +```shell +$ time java Calculator.java +``` diff --git a/python-pycache/example-1/project/calculator.py b/python-pycache/example-1/project/calculator.py new file mode 100644 index 0000000000..c3c7511ddf --- /dev/null +++ b/python-pycache/example-1/project/calculator.py @@ -0,0 +1 @@ +import mathematics.geometry # noqa diff --git a/python-pycache/example-1/project/mathematics/__init__.py b/python-pycache/example-1/project/mathematics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-pycache/example-1/project/mathematics/arithmetic/__init__.py b/python-pycache/example-1/project/mathematics/arithmetic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-pycache/example-1/project/mathematics/arithmetic/add.py b/python-pycache/example-1/project/mathematics/arithmetic/add.py new file mode 100644 index 0000000000..4693ad3cf8 --- /dev/null +++ b/python-pycache/example-1/project/mathematics/arithmetic/add.py @@ -0,0 +1,2 @@ +def add(a, b): + return a + b diff --git a/python-pycache/example-1/project/mathematics/arithmetic/sub.py b/python-pycache/example-1/project/mathematics/arithmetic/sub.py new file mode 100644 index 0000000000..312b477d18 --- /dev/null +++ b/python-pycache/example-1/project/mathematics/arithmetic/sub.py @@ -0,0 +1,2 @@ +def sub(a, b): + return a - b diff --git a/python-pycache/example-1/project/mathematics/geometry/__init__.py b/python-pycache/example-1/project/mathematics/geometry/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-pycache/example-1/project/mathematics/geometry/shapes.py b/python-pycache/example-1/project/mathematics/geometry/shapes.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-pycache/example-2/arithmetic.py b/python-pycache/example-2/arithmetic.py new file mode 100644 index 0000000000..4693ad3cf8 --- /dev/null +++ b/python-pycache/example-2/arithmetic.py @@ -0,0 +1,2 @@ +def add(a, b): + return a + b diff --git a/python-pycache/example-2/calculator.py b/python-pycache/example-2/calculator.py new file mode 100644 index 0000000000..aa4d4851e5 --- /dev/null +++ b/python-pycache/example-2/calculator.py @@ -0,0 +1,3 @@ +from arithmetic import add + +add(3, 4) diff --git a/python-pycache/example-3/project/arithmetic/__init__.py b/python-pycache/example-3/project/arithmetic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-pycache/example-3/project/arithmetic/add.py b/python-pycache/example-3/project/arithmetic/add.py new file mode 100644 index 0000000000..db851d1db4 --- /dev/null +++ b/python-pycache/example-3/project/arithmetic/add.py @@ -0,0 +1,2 @@ +def add_function(a, b): + return a + b diff --git a/python-pycache/example-3/project/arithmetic/sub.py b/python-pycache/example-3/project/arithmetic/sub.py new file mode 100644 index 0000000000..7f77926bc7 --- /dev/null +++ b/python-pycache/example-3/project/arithmetic/sub.py @@ -0,0 +1,2 @@ +def sub_function(a, b): + return a - b diff --git a/python-pycache/example-3/project/calculator.py b/python-pycache/example-3/project/calculator.py new file mode 100644 index 0000000000..4513ca1b25 --- /dev/null +++ b/python-pycache/example-3/project/calculator.py @@ -0,0 +1,3 @@ +import arithmetic.add # noqa +from arithmetic import add # noqa +from arithmetic.add import add_function # noqa diff --git a/python-pycache/pyclean.ps1 b/python-pycache/pyclean.ps1 new file mode 100755 index 0000000000..fc8219907d --- /dev/null +++ b/python-pycache/pyclean.ps1 @@ -0,0 +1 @@ +Get-ChildItem -Path . -Filter __pycache__ -Recurse -Directory | Remove-Item -Recurse -Force diff --git a/python-pycache/pyclean.sh b/python-pycache/pyclean.sh new file mode 100755 index 0000000000..3b79426d0a --- /dev/null +++ b/python-pycache/pyclean.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/python-pycache/xray.py b/python-pycache/xray.py new file mode 100644 index 0000000000..b70c87291b --- /dev/null +++ b/python-pycache/xray.py @@ -0,0 +1,51 @@ +import marshal +from datetime import datetime, timezone +from importlib.util import MAGIC_NUMBER +from pathlib import Path +from pprint import pp +from py_compile import PycInvalidationMode +from sys import argv +from types import SimpleNamespace + + +def main(path): + metadata, code = load_pyc(path) + pp(vars(metadata)) + if metadata.magic_number == MAGIC_NUMBER: + exec(code, globals()) + else: + print("Bytecode incompatible with this interpreter") + + +def load_pyc(path): + with Path(path).open(mode="rb") as file: + return ( + parse_header(file.read(16)), + marshal.loads(file.read()), + ) + + +def parse_header(header): + metadata = SimpleNamespace() + metadata.magic_number = header[0:4] + metadata.magic_int = int.from_bytes(header[0:4][:2], "little") + metadata.python_version = f"3.{(metadata.magic_int - 2900) // 50}" + metadata.bit_field = int.from_bytes(header[4:8], "little") + metadata.pyc_type = { + 0: PycInvalidationMode.TIMESTAMP, + 1: PycInvalidationMode.UNCHECKED_HASH, + 3: PycInvalidationMode.CHECKED_HASH, + }.get(metadata.bit_field) + if metadata.pyc_type is PycInvalidationMode.TIMESTAMP: + metadata.timestamp = datetime.fromtimestamp( + int.from_bytes(header[8:12], "little"), + timezone.utc, + ) + metadata.file_size = int.from_bytes(header[12:16], "little") + else: + metadata.hash_value = header[8:16] + return metadata + + +if __name__ == "__main__": + main(argv[1])