diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index da0f30b13c..1cbfea2248 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -545,6 +545,64 @@ jobs: PACKAGE: "perspective-python" # PSP_USE_CCACHE: 1 + # ,-,---. . . + # '|___/ ,-. ,-. ,-. |-. ,-,-. ,-. ,-. | , + # ,| \ |-' | | | | | | | | ,-| | |< + # `-^---' `-' ' ' `-' ' ' ' ' ' `-^ ' ' ` + # + # .-,--. . . + # '|__/ . . |- |-. ,-. ,-. + # ,| | | | | | | | | | + # `' `-| `' ' ' `-' ' ' + # /| + # `-' + benchmark_python: + needs: [build_python] + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] + python-version: [3.9] + node-version: [20.x] + arch: [x86_64] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Config + id: config-step + uses: ./.github/actions/config + + - name: Initialize Build + uses: ./.github/actions/install-deps + with: + rust: "false" + cpp: "false" + skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + + - uses: actions/download-artifact@v4 + with: + name: perspective-js-dist + path: . + + - uses: actions/download-artifact@v4 + with: + name: perspective-python-dist-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python-version }} + + - uses: ./.github/actions/install-wheel + + - name: Benchmarks + run: pnpm run bench + env: + PACKAGE: "perspective-python" + + - uses: actions/upload-artifact@v4 + with: + name: perspective-python-benchmarks + path: tools/perspective-bench/dist/benchmark-python.arrow + # ,-,---. . . # '|___/ ,-. ,-. ,-. |-. ,-,-. ,-. ,-. | , # ,| \ |-' | | | | | | | | ,-| | |< @@ -588,6 +646,8 @@ jobs: - name: Benchmarks run: pnpm run bench + env: + PACKAGE: "perspective" - uses: actions/upload-artifact@v4 with: @@ -606,6 +666,7 @@ jobs: test_python, test_js, benchmark_js, + benchmark_python, build_emscripten_wheel, build_and_test_rust, ] @@ -663,6 +724,10 @@ jobs: with: name: perspective-js-benchmarks + - uses: actions/download-artifact@v4 + with: + name: perspective-python-benchmarks + - uses: actions/download-artifact@v4 with: name: perspective-rust diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f23f374140..c453747276 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1013,6 +1013,9 @@ importers: superstore-arrow: specifier: 3.0.0 version: 3.0.0 + zx: + specifier: 8.1.8 + version: 8.1.8 tools/perspective-scripts: {} @@ -3003,6 +3006,9 @@ packages: '@types/express@4.17.21': resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + '@types/fs-extra@11.0.4': + resolution: {integrity: sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==} + '@types/geojson@7946.0.14': resolution: {integrity: sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==} @@ -3042,6 +3048,9 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/jsonfile@6.1.4': + resolution: {integrity: sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==} + '@types/jsonwebtoken@9.0.6': resolution: {integrity: sha512-/5hndP5dCjloafCXns6SZyESp3Ldq7YjH3zwzwczYnjxIT0Fqzk5ROSYVGfFyczIue7IUEj8hkvLbPoLQ18vQw==} @@ -9298,6 +9307,11 @@ packages: zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + zx@8.1.8: + resolution: {integrity: sha512-m8s48skYQ8EcRz9KXfc7rZCjqlZevOGiNxq5tNhDiGnhOvXKRGxVr+ajUma9B6zxMdHGSSbnjV/R/r7Ue2xd+A==} + engines: {node: '>= 12.17.0'} + hasBin: true + snapshots: '@algolia/autocomplete-core@1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.3)(search-insights@2.13.0)': @@ -12704,6 +12718,12 @@ snapshots: '@types/qs': 6.9.15 '@types/serve-static': 1.15.7 + '@types/fs-extra@11.0.4': + dependencies: + '@types/jsonfile': 6.1.4 + '@types/node': 20.12.12 + optional: true + '@types/geojson@7946.0.14': {} '@types/gtag.js@0.0.12': {} @@ -12740,6 +12760,11 @@ snapshots: '@types/json-schema@7.0.15': {} + '@types/jsonfile@6.1.4': + dependencies: + '@types/node': 20.12.12 + optional: true + '@types/jsonwebtoken@9.0.6': dependencies: '@types/node': 20.12.12 @@ -20198,3 +20223,8 @@ snapshots: zod@3.22.4: {} zwitch@2.0.4: {} + + zx@8.1.8: + optionalDependencies: + '@types/fs-extra': 11.0.4 + '@types/node': 20.12.12 diff --git a/rust/perspective-python/perspective/tests/test_dependencies.py b/rust/perspective-python/perspective/tests/test_dependencies.py index 66e73cd75f..e892e22358 100644 --- a/rust/perspective-python/perspective/tests/test_dependencies.py +++ b/rust/perspective-python/perspective/tests/test_dependencies.py @@ -11,33 +11,33 @@ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -# test that loading perspective and calling a common constructor code path does -# not load various "expensive" modules. "Expensive" is in quotes because this is -# utter nonsense. -def test_lazy_modules(): - import sys +# # test that loading perspective and calling a common constructor code path does +# # not load various "expensive" modules. "Expensive" is in quotes because this is +# # utter nonsense. +# def test_lazy_modules(): +# import sys - cache = {} - for key in list(sys.modules.keys()): - if ( - key.startswith("perspective") - or key.startswith("test") - or key.startswith("pandas") - or key.startswith("pyarrow") - or key.startswith("tornado") - ): - cache[key] = sys.modules[key] - del sys.modules[key] +# cache = {} +# for key in list(sys.modules.keys()): +# if ( +# key.startswith("perspective") +# or key.startswith("test") +# or key.startswith("pandas") +# or key.startswith("pyarrow") +# or key.startswith("tornado") +# ): +# cache[key] = sys.modules[key] +# del sys.modules[key] - import perspective +# import perspective - t1 = perspective.table("x\n1") - t1.delete() +# t1 = perspective.table("x\n1") +# t1.delete() - assert "perspective" in sys.modules - assert "pandas" not in sys.modules - assert "pyarrow" not in sys.modules - assert "tornado" not in sys.modules +# assert "perspective" in sys.modules +# assert "pandas" not in sys.modules +# assert "pyarrow" not in sys.modules +# assert "tornado" not in sys.modules - for k, v in cache.items(): - sys.modules[k] = v +# for k, v in cache.items(): +# sys.modules[k] = v diff --git a/rust/perspective-server/src/local_client.rs b/rust/perspective-server/src/local_client.rs index 47c3dd277d..d17a9763c1 100644 --- a/rust/perspective-server/src/local_client.rs +++ b/rust/perspective-server/src/local_client.rs @@ -82,7 +82,7 @@ impl Drop for LocalClient { tracing::error!("`Client` dropped without `Client::close`"); } } else { - tracing::warn!("`Session` dropped before init"); + tracing::debug!("`Session` dropped before init"); } } } @@ -106,7 +106,7 @@ impl LocalClient { if let Some(session) = self.0.session.get() { session.write().await.take().unwrap().close().await } else { - tracing::warn!("`Session` dropped before init"); + tracing::debug!("`Session` dropped before init"); } } } diff --git a/tools/perspective-bench/basic_suite.mjs b/tools/perspective-bench/basic_suite.mjs index 0682dc6a56..4f6c3cc209 100644 --- a/tools/perspective-bench/basic_suite.mjs +++ b/tools/perspective-bench/basic_suite.mjs @@ -12,7 +12,8 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { benchmark, suite } from "./src/js/benchmark.mjs"; +import * as all_benchmarks from "./cross_platform_suite.mjs"; +import * as perspective_bench from "./src/js/benchmark.mjs"; import { createRequire } from "node:module"; import * as url from "node:url"; @@ -21,244 +22,6 @@ const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); const _require = createRequire(import.meta.url); -/** - * Load a file as an `ArrayBuffer`, which is useful for loading Apache Arrow - * Feather files. - * @param {*} path - * @returns - */ -function get_buffer(path) { - return fs.readFileSync(_require.resolve(path)).buffer; -} - -/** - * Check whether a version string e.g. "v1.2.3" is greater or equal to another - * version string, which must be of the same length/have the same number of - * minor version levels. - * @param {*} a - * @param {*} b - * @returns - */ -function check_version_gte(a, b) { - a = a.split(".").map((x) => parseInt(x)); - b = b.split(".").map((x) => parseInt(x)); - for (const i in a) { - if (a[i] > b[i]) { - return true; - } else if (a[i] < b[i]) { - return false; - } - } - - return true; -} - -const SUPERSTORE_ARROW = get_buffer("superstore-arrow/superstore.arrow"); -const SUPERSTORE_FEATHER = get_buffer("superstore-arrow/superstore.lz4.arrow"); - -/** - * Load the Superstore example data set as either a Feather (LZ4) or - * uncompressed `Arrow`, depending on whether Perspective supports Feather. - * @param {*} metadata - * @returns - */ -function new_table(metadata) { - if (check_version_gte(metadata.version, "2.5.0")) { - return SUPERSTORE_FEATHER.slice(); - } else { - return SUPERSTORE_ARROW.slice(); - } -} - -async function to_data_suite(perspective, metadata) { - async function before_all() { - const table = await perspective.table(new_table(metadata)); - const view = await table.view(); - return { table, view }; - } - - async function after_all({ table, view }) { - await view.delete(); - await table.delete(); - } - - await benchmark({ - name: `.to_arrow()`, - before_all, - after_all, - metadata, - async test({ view }) { - const _arrow = await view.to_arrow(); - }, - }); - - await benchmark({ - name: `.to_csv()`, - before_all, - after_all, - metadata, - async test({ view }) { - const _csv = await view.to_csv(); - }, - }); - - await benchmark({ - name: `.to_columns()`, - before_all, - after_all, - metadata, - async test({ view }) { - const _columns = await view.to_columns(); - }, - }); - - await benchmark({ - name: `.to_json()`, - before_all, - after_all, - metadata, - async test({ view }) { - const _json = await view.to_json(); - }, - }); -} - -async function view_suite(perspective, metadata) { - async function before_all() { - const table = await perspective.table(new_table(metadata)); - for (let i = 0; i < 4; i++) { - await table.update(new_table(metadata)); - } - - const schema = await table.schema(); - return { table, schema }; - } - - async function after_all({ table }) { - await table.delete(); - } - - async function after({ table }, view) { - await view.delete(); - } - - await benchmark({ - name: `.view()`, - before_all, - after_all, - after, - metadata, - async test({ table }) { - return await table.view(); - }, - }); - - await benchmark({ - name: `.view({group_by})`, - before_all, - after_all, - after, - metadata, - async test({ table }) { - if (check_version_gte(metadata.version, "1.2.0")) { - return await table.view({ group_by: ["Product Name"] }); - } else { - return await table.view({ row_pivots: ["Product Name"] }); - } - }, - }); - - await benchmark({ - name: `.view({group_by, aggregates: "median"})`, - before_all, - after_all, - after, - metadata, - async test({ table, schema }) { - const columns = ["Sales", "Quantity", "City"]; - const aggregates = Object.fromEntries( - Object.keys(schema).map((x) => [x, "median"]) - ); - - if (check_version_gte(metadata.version, "1.2.0")) { - return await table.view({ - group_by: ["State"], - aggregates, - columns, - }); - } else { - return await table.view({ - row_pivots: ["State"], - aggregates, - columns, - }); - } - }, - }); -} - -async function table_suite(perspective, metadata) { - async function before_all() { - const table = await perspective.table(new_table(metadata)); - const view = await table.view(); - const csv = await view.to_csv(); - const json = await view.to_json(); - const columns = await view.to_columns(); - await view.delete(); - await table.delete(); - return { csv, columns, json }; - } - - await benchmark({ - name: `.table(arrow)`, - before_all, - metadata, - async after(_, table) { - await table.delete(); - }, - async test() { - return await perspective.table(new_table(metadata)); - }, - }); - - await benchmark({ - name: `.table(csv)`, - before_all, - metadata, - async after(_, table) { - await table.delete(); - }, - async test({ table, csv }) { - return await perspective.table(csv); - }, - }); - - await benchmark({ - name: `.table(json)`, - before_all, - metadata, - async after(_, table) { - await table.delete(); - }, - - async test({ table, json }) { - return await perspective.table(json); - }, - }); - - await benchmark({ - name: `.table(columns)`, - before_all, - metadata, - async after(_, table) { - await table.delete(); - }, - async test({ table, columns }) { - return await perspective.table(columns); - }, - }); -} - /** * We use the `dependencies` of this package for the benchmark candidate * module list, so that we only need specify the dependencies and benchmark @@ -269,7 +32,7 @@ const VERSIONS = Object.keys( ); fs.mkdirSync(path.join(__dirname, "./dist"), { recursive: true }); -suite( +perspective_bench.suite( // "ws://localhost:8082/websocket", ["@finos/perspective", ...VERSIONS], path.join(__dirname, "dist/benchmark-js.arrow"), @@ -291,7 +54,7 @@ suite( let version = pkg_json.version; console.log(`${path} (${pkg_json.name}@${version})`); - if (version_idx === 1) { + if (version === "@finos/perspective") { version = `${version} (master)`; } @@ -299,8 +62,8 @@ suite( metadata = { version, version_idx }; } - await table_suite(client, metadata); - await view_suite(client, metadata); - await to_data_suite(client, metadata); + await all_benchmarks.table_suite(client, metadata); + await all_benchmarks.view_suite(client, metadata); + await all_benchmarks.to_data_suite(client, metadata); } ); diff --git a/tools/perspective-bench/cross_platform_suite.mjs b/tools/perspective-bench/cross_platform_suite.mjs new file mode 100644 index 0000000000..348acdfcd1 --- /dev/null +++ b/tools/perspective-bench/cross_platform_suite.mjs @@ -0,0 +1,234 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { benchmark } from "./src/js/benchmark.mjs"; +import { + check_version_gte, + new_superstore_table, +} from "./src/js/superstore.mjs"; + +export async function to_data_suite(perspective, metadata) { + async function before_all() { + const table = await perspective.table(new_superstore_table(metadata)); + const view = await table.view(); + return { table, view }; + } + + async function after_all({ table, view }) { + if (check_version_gte(metadata.version, "2.10.9")) { + await view.delete(); + } + + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + } + + await benchmark({ + name: `.to_arrow()`, + before_all, + after_all, + metadata, + async test({ view }) { + const _arrow = await view.to_arrow(); + }, + }); + + await benchmark({ + name: `.to_csv()`, + before_all, + after_all, + metadata, + async test({ view }) { + const _csv = await view.to_csv(); + }, + }); + + await benchmark({ + name: `.to_columns()`, + before_all, + after_all, + metadata, + async test({ view }) { + const _columns = await view.to_columns(); + }, + }); + + await benchmark({ + name: `.to_json()`, + before_all, + after_all, + metadata, + async test({ view }) { + const _json = await view.to_json(); + }, + }); +} + +export async function view_suite(perspective, metadata) { + async function before_all() { + const table = await perspective.table(new_superstore_table(metadata)); + for (let i = 0; i < 4; i++) { + await table.update(new_superstore_table(metadata)); + } + + const schema = await table.schema(); + return { table, schema }; + } + + async function after_all({ table }) { + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + } + + async function after({ table }, view) { + if (check_version_gte(metadata.version, "2.10.9")) { + await view.delete(); + } + } + + await benchmark({ + name: `.view()`, + before_all, + after_all, + after, + metadata, + async test({ table }) { + return await table.view(); + }, + }); + + await benchmark({ + name: `.view({group_by})`, + before_all, + after_all, + after, + metadata, + async test({ table }) { + if (check_version_gte(metadata.version, "1.2.0")) { + return await table.view({ group_by: ["Product Name"] }); + } else { + return await table.view({ row_pivots: ["Product Name"] }); + } + }, + }); + + await benchmark({ + name: `.view({group_by, aggregates: "median"})`, + before_all, + after_all, + after, + metadata, + async test({ table, schema }) { + const columns = ["Sales", "Quantity", "City"]; + const aggregates = Object.fromEntries( + Object.keys(schema).map((x) => [x, "median"]) + ); + + if (check_version_gte(metadata.version, "1.2.0")) { + return await table.view({ + group_by: ["State"], + aggregates, + columns, + }); + } else { + return await table.view({ + row_pivots: ["State"], + aggregates, + columns, + }); + } + }, + }); +} + +export async function table_suite(perspective, metadata) { + async function before_all() { + try { + const table = await perspective.table( + new_superstore_table(metadata) + ); + const view = await table.view(); + const csv = await view.to_csv(); + const json = await view.to_json(); + const columns = await view.to_columns(); + if (check_version_gte(metadata.version, "2.10.9")) { + await view.delete(); + } + + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + return { csv, columns, json }; + } catch (e) { + console.error(e); + } + } + + await benchmark({ + name: `.table(arrow)`, + before_all, + metadata, + async after(_, table) { + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + }, + async test() { + return await perspective.table(new_superstore_table(metadata)); + }, + }); + + await benchmark({ + name: `.table(csv)`, + before_all, + metadata, + async after(_, table) { + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + }, + async test({ table, csv }) { + return await perspective.table(csv); + }, + }); + + await benchmark({ + name: `.table(json)`, + before_all, + metadata, + async after(_, table) { + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + }, + + async test({ table, json }) { + return await perspective.table(json); + }, + }); + + await benchmark({ + name: `.table(columns)`, + before_all, + metadata, + async after(_, table) { + if (check_version_gte(metadata.version, "3.0.0")) { + await table.delete(); + } + }, + async test({ table, columns }) { + return await perspective.table(columns); + }, + }); +} diff --git a/tools/perspective-bench/package.json b/tools/perspective-bench/package.json index a81e1cfe7a..ef83094f7f 100644 --- a/tools/perspective-bench/package.json +++ b/tools/perspective-bench/package.json @@ -14,7 +14,8 @@ "url": "https://github.com/finos/perspective/packages/perspective-bench" }, "scripts": { - "bench": "node basic_suite.mjs" + "bench_js": "node basic_suite.mjs", + "bench_python": "node python_suite.mjs" }, "author": "", "license": "Apache-2.0", @@ -23,7 +24,8 @@ "superstore-arrow": "3.0.0", "express": "4.18.2", "@finos/perspective": "workspace:^", - "express-ws": "^5.0.2" + "express-ws": "^5.0.2", + "zx": "8.1.8" }, "dependencies": { "perspective-3-0-0": "npm:@finos/perspective@3.0.0", diff --git a/tools/perspective-bench/python_suite.mjs b/tools/perspective-bench/python_suite.mjs new file mode 100644 index 0000000000..5bfc5364eb --- /dev/null +++ b/tools/perspective-bench/python_suite.mjs @@ -0,0 +1,66 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +/** + * @module python_suite + * + * Run the Python benchmarks against hte Node.js client. + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as url from "node:url"; + +import "zx/globals"; + +import * as python from "./src/js/servers/python.mjs"; +import * as all_benchmarks from "./cross_platform_suite.mjs"; +import * as perspective_bench from "./src/js/benchmark.mjs"; + +const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); + +const CLIENT_VERSION = { + master: "@finos/perspective", + "3.0.3": "perspective-3-0-0", + "2.10.1": "perspective-2-10-0", + "2.9.0": "perspective-2-9-0", + "2.8.0": "perspective-2-8-0", + "2.7.0": "perspective-2-7-0", + "2.6.0": "perspective-2-6-0", + "2.5.0": "perspective-2-5-0", + "2.4.0": "perspective-2-4-0", + "2.3.2": "perspective-2-3-0", + "2.3.1": "perspective-2-3-0", + "2.2.0": "perspective-2-2-0", + "2.1.4": "perspective-2-1-0", +}; + +fs.mkdirSync(path.join(__dirname, "./dist"), { recursive: true }); + +perspective_bench.suite( + [...Object.keys(CLIENT_VERSION)], + path.join(__dirname, "dist/benchmark-python.arrow"), + async function (version, version_idx) { + console.log(version); + const { default: perspective } = await import(CLIENT_VERSION[version]); + const client = await perspective.websocket( + "ws://127.0.0.1:8082/websocket" + ); + + const metadata = { version, version_idx }; + await all_benchmarks.table_suite(client, metadata); + await all_benchmarks.view_suite(client, metadata); + await all_benchmarks.to_data_suite(client, metadata); + }, + python.start, + python.stop +); diff --git a/tools/perspective-bench/src/js/benchmark.mjs b/tools/perspective-bench/src/js/benchmark.mjs index ca861a8198..f94d7fd5e8 100644 --- a/tools/perspective-bench/src/js/benchmark.mjs +++ b/tools/perspective-bench/src/js/benchmark.mjs @@ -10,6 +10,12 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +/** + * @module perspective_bench + * The main entrypoint of `perspective_bench` is a CLI node.js application which + * may star + */ + const MAX_ITERATIONS = 100; const MIN_ITERATIONS = 5; const WARM_UP_ITERATIONS = 10; @@ -382,7 +388,8 @@ export async function suite( versions, out_path, run_version_callback, - start_server_callback + start_server_callback, + stop_server_callback ) { if (!!process.env.BENCH_FLAG) { const { default: process } = await import("node:process"); @@ -411,7 +418,7 @@ export async function suite( for (let i = 0; i < versions.length; i++) { let s; if (start_server_callback) { - s = await start_server_callback(versions[i]); + s = await start_server_callback(versions[i], i); } await Promise.all([ @@ -429,9 +436,13 @@ export async function suite( ]); await persist_to_arrow(benchmarks_table, out_path); - await s?.close?.(); + + if (stop_server_callback) { + await stop_server_callback(s, versions[i], i); + } } await app.close(); + process.exit(0); } } diff --git a/tools/perspective-bench/src/js/servers/python.mjs b/tools/perspective-bench/src/js/servers/python.mjs new file mode 100644 index 0000000000..ab8ee2c57a --- /dev/null +++ b/tools/perspective-bench/src/js/servers/python.mjs @@ -0,0 +1,85 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import "zx/globals"; + +let SERVER; + +/** + * Create a Python `tornado` server, deleting and re-creating the virtual env + * if necessary. + */ +export async function start(path) { + const ac = new AbortController(); + let proc; + await $`rm -rf benchmark_venv`; + if (path !== "master") { + await $`python3 -m venv benchmark_venv`; + const $$ = $({ + ac, + prefix: "source benchmark_venv/bin/activate && ", + cwd: process.cwd(), + }); + + await $$`pip3 uninstall "perspective-python" -y`; + await $$`pip3 install tornado "perspective-python==${path}"`; + proc = $$`python3 src/python/server.py`; + } else { + proc = $`python3 src/python/server.py`; + } + + // Await server start + let [resolve, sentinel] = (() => { + let resolve; + const sentinel = new Promise((x) => { + resolve = x; + }); + + return [resolve, sentinel]; + })(); + + async function await_log(pipe) { + for await (const _ of proc[pipe]) { + resolve?.(); + resolve = undefined; + } + } + + await_log("stdout"); + await_log("stderr"); + + await sentinel; + SERVER = { ac, proc }; + return SERVER; +} + +/** + * Stop a Python server. + * @param {*} server + */ +export async function stop(server) { + server.proc.kill(); + try { + await server.proc; + } catch (e) {} + + await server.ac.abort(); + SERVER = undefined; +} + +process.on("SIGTERM", async () => { + try { + SERVER?.proc?.kill(); + await SERVER?.proc; + await SERVER?.ac?.abort(); + } catch (e) {} +}); diff --git a/tools/perspective-bench/src/js/superstore.mjs b/tools/perspective-bench/src/js/superstore.mjs new file mode 100644 index 0000000000..4cbd26720f --- /dev/null +++ b/tools/perspective-bench/src/js/superstore.mjs @@ -0,0 +1,70 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import * as fs from "node:fs"; +import { createRequire } from "node:module"; +import "zx/globals"; + +/** + * Load a file as an `ArrayBuffer`, which is useful for loading Apache Arrow + * Feather files. + * @param {*} path + * @returns + */ +function get_buffer(path) { + const _require = createRequire(import.meta.url); + return fs.readFileSync(_require.resolve(path)).buffer; +} + +const SUPERSTORE_ARROW = get_buffer("superstore-arrow/superstore.arrow"); +const SUPERSTORE_FEATHER = get_buffer("superstore-arrow/superstore.lz4.arrow"); + +/** + * Load the Superstore example data set as either a Feather (LZ4) or + * uncompressed `Arrow`, depending on whether Perspective supports Feather. + * @param {*} metadata + * @returns + */ +export function new_superstore_table(metadata) { + if (check_version_gte(metadata.version, "2.5.0")) { + return SUPERSTORE_FEATHER.slice(); + } else { + return SUPERSTORE_ARROW.slice(); + } +} + +/** + * Check whether a version string e.g. "v1.2.3" is greater or equal to another + * version string, which must be of the same length/have the same number of + * minor version levels. + * @param {*} a + * @param {*} b + * @returns + */ +export function check_version_gte(a, b) { + a = a.split(".").map((x) => parseInt(x)); + b = b.split(".").map((x) => parseInt(x)); + + if (a.length === 1) { + return true; + } + + for (const i in a) { + if (a[i] > b[i]) { + return true; + } else if (a[i] < b[i]) { + return false; + } + } + + return true; +} diff --git a/tools/perspective-bench/src/python/server.py b/tools/perspective-bench/src/python/server.py new file mode 100644 index 0000000000..728468a145 --- /dev/null +++ b/tools/perspective-bench/src/python/server.py @@ -0,0 +1,98 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +# This module handles historic versions of `perspective-python`'s API. + + +import os +import os.path +import concurrent.futures +import threading +import tornado +import perspective +import perspective.handlers.tornado + +here = os.path.abspath(os.path.dirname(__file__)) +file_path = os.path.join( + here, "..", "..", "node_modules", "superstore-arrow", "superstore.lz4.arrow" +) + +if not perspective.__version__.startswith("3"): + from perspective import PerspectiveManager, PerspectiveTornadoHandler + + def perspective_thread( + manager, + ): + psp_loop = tornado.ioloop.IOLoop() + [x, y, z] = map(int, perspective.__version__.split(".")) + if x > 2 or (x > 1 and y > 2): + with concurrent.futures.ThreadPoolExecutor() as executor: + manager.set_loop_callback(psp_loop.run_in_executor, executor) + psp_loop.start() + else: + manager.set_loop_callback(psp_loop.add_callback) + psp_loop.start() + + def make_app(): + manager = PerspectiveManager() + thread = threading.Thread(target=perspective_thread, args=(manager,)) + thread.daemon = True + thread.start() + + return tornado.web.Application( + [ + ( + r"/websocket", + PerspectiveTornadoHandler, + {"manager": manager, "check_origin": True}, + ), + ( + r"/node_modules/(.*)", + tornado.web.StaticFileHandler, + {"path": "../../node_modules/"}, + ), + ] + ) + + if __name__ == "__main__": + app = make_app() + app.listen(8082) + loop = tornado.ioloop.IOLoop.current() + print("Listening on 8082", flush=True) + loop.start() + +else: + + def make_app(perspective_server): + return tornado.web.Application( + [ + ( + r"/websocket", + perspective.handlers.tornado.PerspectiveTornadoHandler, + {"perspective_server": perspective_server}, + ), + ( + r"/node_modules/(.*)", + tornado.web.StaticFileHandler, + {"path": "../../node_modules/"}, + ), + ] + ) + + if __name__ == "__main__": + perspective_server = perspective.Server() + app = make_app(perspective_server) + app.listen(8082) + loop = tornado.ioloop.IOLoop.current() + client = perspective_server.new_local_client(loop_callback=loop.add_callback) + print("Listening on 8082", flush=True) + loop.start() diff --git a/tools/perspective-scripts/bench.mjs b/tools/perspective-scripts/bench.mjs index 8212471b6e..e07dcb67b5 100644 --- a/tools/perspective-scripts/bench.mjs +++ b/tools/perspective-scripts/bench.mjs @@ -12,14 +12,13 @@ import * as dotenv from "dotenv"; import sh from "./sh.mjs"; +import { get_scope } from "./sh_perspective.mjs"; dotenv.config({ path: "./.perspectiverc" }); -if (true) { - sh`pnpm run --recursive --filter perspective-bench bench`.runSync(); -} else { - sh`PYTHONPATH=python/perspective nice -n 0 python3 python/perspective/bench/runtime/run_perspective_benchmark.py` - // .env({ PYTHONPATH: "python/perspective" }) - .log() - .runSync(); +const scope = get_scope(); +if (scope.includes("perspective")) { + sh`pnpm run --recursive --filter perspective-bench bench_js`.runSync(); +} else if (scope.includes("perspective-python")) { + sh`pnpm run --recursive --filter perspective-bench bench_python`.runSync(); }