Merge pull request #2754 from finos/python-imports

Python API changes from community

cmake/modules/FindInstallDependency.cmake

@@ -48,11 +48,16 @@ function(psp_build_dep name cmake_file)
         set(ARROW_LZ4 ON)
         set(ARROW_WITH_ZSTD ON)
         set(ARROW_WITH_LZ4 ON)
-        set(ARROW_ENABLE_THREADING OFF)
         set(ARROW_NO_EXPORT ON)
-        if(WIN32 AND NOT PSP_BUILD_WASM)
-            set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
+        if(PSP_WASM_BUILD)
+            set(ARROW_ENABLE_THREADING OFF)
+        else()
+            set(ARROW_ENABLE_THREADING ON)
+            if(WIN32)
+                set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
+            endif()
         endif()
+
         include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-build/src)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
             ${CMAKE_BINARY_DIR}/${name}-build

cpp/perspective/src/cpp/arrow_csv.cpp

@@ -609,12 +609,11 @@ csvToTable(
     auto read_options = arrow::csv::ReadOptions::Defaults();
     auto parse_options = arrow::csv::ParseOptions::Defaults();
     auto convert_options = arrow::csv::ConvertOptions::Defaults();
-    // #ifdef PSP_PARALLEL_FOR
-    //         read_options.use_threads = true;
-    // #else
-    //         read_options.use_threads = false;
-    // #endif
-    read_options.use_threads = false;
+    #ifdef PSP_PARALLEL_FOR
+        read_options.use_threads = true;
+    #else
+        read_options.use_threads = false;
+    #endif
     parse_options.newlines_in_values = true;
 
     if (is_update) {

cpp/perspective/src/cpp/table.cpp

@@ -1082,8 +1082,8 @@ Table::from_cols(
 
     if (is_implicit) {
         for (t_uindex ii = 0; ii < nrows; ii++) {
-            psp_pkey_col->set_nth<std::int32_t>(ii, ii);
-            psp_okey_col->set_nth<std::int32_t>(ii, ii);
+            psp_pkey_col->set_nth<std::int32_t>(ii, ii % limit);
+            psp_okey_col->set_nth<std::int32_t>(ii, ii % limit);
         }
     }
 

cpp/perspective/src/cpp/view.cpp

@@ -1211,12 +1211,12 @@ View<CTX_T>::data_slice_to_arrow(
         options.codec = std::move(codec).ValueUnsafe();
     }
 
-    // #ifdef PSP_PARALLEL_FOR
-    //     options.use_threads = false;
-    // #else
-    //     options.use_threads = false;
-    // #endif
-    options.use_threads = false;
+    #ifdef PSP_PARALLEL_FOR
+        options.use_threads = true;
+    #else
+        options.use_threads = false;
+    #endif
+
     auto res = arrow::ipc::MakeStreamWriter(&sink, arrow_schema, options);
     std::shared_ptr<arrow::ipc::RecordBatchWriter> writer = *res;
     PSP_CHECK_ARROW_STATUS(writer->WriteRecordBatch(*batches));

examples/python-tornado-streaming/server.py

@@ -102,7 +102,7 @@ def make_app(perspective_server):
         [
             (
                 r"/websocket",
-                perspective.PerspectiveTornadoHandler,
+                perspective.handlers.tornado.PerspectiveTornadoHandler,
                 {"perspective_server": perspective_server},
             ),
             (

examples/python-tornado/server.py

@@ -18,6 +18,7 @@
 import tornado.ioloop
 
 import perspective
+import perspective.handlers.tornado
 
 here = os.path.abspath(os.path.dirname(__file__))
 file_path = os.path.join(

examples/rust-axum/src/main.rs

@@ -18,7 +18,6 @@ use axum::routing::get_service;
 use axum::Router;
 use perspective::client::{TableInitOptions, UpdateData};
 use perspective::server::Server;
-use perspective::LocalClient;
 use tower_http::services::{ServeDir, ServeFile};
 use tower_http::trace::TraceLayer;
 use tracing_subscriber::filter::LevelFilter;
@@ -36,7 +35,7 @@ type AppError = Box<dyn std::error::Error + Send + Sync>;
 /// Load the example Apache Arrow file from disk and create a
 /// [`perspective::Table`] named "my_data_source".
 async fn load_server_arrow(server: &Server) -> Result<(), AppError> {
-    let client = LocalClient::new(server);
+    let client = server.new_local_client();
     let mut file = File::open(std::path::Path::new(ROOT_PATH).join(ARROW_FILE_PATH))?;
     let mut feather = Vec::with_capacity(file.metadata()?.len() as usize);
     file.read_to_end(&mut feather)?;

rust/bootstrap/main.rs

@@ -62,7 +62,7 @@ fn main() {
         .args(debug_flags)
         .env(
             "BOOTSTRAP_TARGET",
-            &fs::canonicalize(args.input.clone()).unwrap(),
+            fs::canonicalize(args.input.clone()).unwrap(),
         )
         .execute();
 

rust/perspective-python/build.mjs

@@ -13,6 +13,7 @@
 import * as fs from "node:fs";
 import sh from "../../tools/perspective-scripts/sh.mjs";
 import * as url from "url";
+import * as TOML from "@iarna/toml";
 
 const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1);
 const pkg = JSON.parse(
@@ -89,7 +90,13 @@ if (build_wheel) {
     cmd.sh(`maturin build ${flags} --features=external-cpp ${target}`);
 }
 
+const old = fs.readFileSync("./pyproject.toml");
+
 if (build_sdist) {
+    const toml = TOML.parse(old);
+    console.log(toml);
+    delete toml.tool.maturin["data"];
+    fs.writeFileSync("./pyproject.toml", TOML.stringify(toml));
     cmd.sh(`maturin sdist`);
 }
 
@@ -98,3 +105,7 @@ if (!build_wheel && !build_sdist) {
 }
 
 cmd.runSync();
+
+if (build_sdist) {
+    fs.writeFileSync("./pyproject.toml", old);
+}

rust/perspective-python/docs/lib.md

@@ -51,26 +51,23 @@ makes it simple to extend a Tornado server with virtual Perspective support.
 
 The `perspective` module exports several tools:
 
--   `PerspectiveWidget` the JupyterLab widget for interactive visualization.
--   Perspective webserver handlers that interface seamlessly with
-    `<perspective-viewer>` in JavaScript.
-    -   `PerspectiveTornadoHandler` for [Tornado](https://www.tornadoweb.org/)
-    -   `PerspectiveStarletteHandler` for [Starlette](https://www.starlette.io/)
-        and [FastAPI](https://fastapi.tiangolo.com)
-    -   `PerspectiveAIOHTTPHandler` for [AIOHTTP](https://docs.aiohttp.org),
-    -   `tornado_websocket`, a Tornado-based websocket client
-    -   `aiohttp_websocket` an AIOHTTP-based websocket client
--   `Server` the session manager for a shared server deployment of
-    `perspective-python`.
+-   `Server` the constructor for a new isntance of the Perspective data engine.
+-   The `perspective.widget` module exports `PerspectiveWidget`, the JupyterLab
+    widget for interactive visualization in a notebook cell.
+-   The `perspective.handlers` modules exports web frameworks handlers that
+    interface with a `perspective-client` in JavaScript.
+    -   `perspective.handlers.tornado.PerspectiveTornadoHandler` for
+        [Tornado](https://www.tornadoweb.org/)
+    -   `perspective.handlers.starlette.PerspectiveStarletteHandler` for
+        [Starlette](https://www.starlette.io/) and
+        [FastAPI](https://fastapi.tiangolo.com)
+    -   `perspective.handlers.aiohttp.PerspectiveAIOHTTPHandler` for
+        [AIOHTTP](https://docs.aiohttp.org),
 
 This user's guide provides an overview of the most common ways to use
 Perspective in Python: the `Table` API, the JupyterLab widget, and the Tornado
 handler.
 
-For an understanding of Perspective's core concepts, see the [Table](table.md),
-[View](view.md), and [Data Binding](server.md) documentation. For API
-documentation, see the [Python API](obj/perspective-python.md).
-
 [More Examples](https://github.com/finos/perspective/tree/master/examples) are
 available on GitHub.
 
@@ -80,10 +77,6 @@ available on GitHub.
 widget, and a WebSocket handlers for several webserver libraries that allow you
 to host Perspective using server-side Python.
 
-In addition to supporting row/columnar formats of data using `dict` and `list`,
-`pandas.DataFrame`, dictionaries of NumPy arrays, NumPy structured arrays, and
-NumPy record arrays are all supported in `perspective-python`.
-
 ### PyPI
 
 `perspective-python` can be installed from [PyPI](https://pypi.org) via `pip`:
@@ -146,15 +139,15 @@ data = pd.DataFrame({
     "string": [str(i) for i in range(100)]
 })
 
-table = perspective.Table(data, index="float")
+table = perspective.table(data, index="float")
 ```
 
 Likewise, a `View` can be created via the `view()` method:
 
 ```python
 view = table.view(group_by=["float"], filter=[["bool", "==", True]])
-column_data = view.to_dict()
-row_data = view.to_records()
+column_data = view.to_columns()
+row_data = view.to_json()
 ```
 
 ### Pandas Support
@@ -198,14 +191,14 @@ def update_callback():
     print("Updated!")
 
 # set the update callback
-view.on_update(update_callback)
+on_update_id = view.on_update(update_callback)
 
 
 def delete_callback():
     print("Deleted!")
 
 # set the delete callback
-view.on_delete(delete_callback)
+on_delete_id = view.on_delete(delete_callback)
 
 # set a lambda as a callback
 view.on_delete(lambda: print("Deleted x2!"))
@@ -215,16 +208,16 @@ If the callback is a named reference to a function, it can be removed with
 `remove_update` or `remove_delete`:
 
 ```python
-view.remove_update(update_callback)
-view.remove_delete(delete_callback)
+view.remove_update(on_update_id)
+view.remove_delete(on_delete_id)
 ```
 
 Callbacks defined with a lambda function cannot be removed, as lambda functions
 have no identifier.
 
 #### Multi-threading
 
-Perspective's server is completely releases the GIL (though it may be retained
+Perspective's server API releases the GIL when called (though it may be retained
 for some portion of the `Client` call to encode RPC messages). It also
 dispatches to an internal thread pool for some operations, enabling better
 parallelism and overall better server performance. However, Perspective's Python

rust/perspective-python/package.json

@@ -16,6 +16,7 @@
     },
     "devDependencies": {
         "@finos/perspective-scripts": "workspace:*",
+        "@iarna/toml": "^1.0.0-rc.1",
         "cpy": "^9.0.1"
     },
     "dependencies": {

rust/perspective-python/perspective/__init__.py

@@ -13,77 +13,40 @@
 __version__ = "3.0.3"
 __all__ = [
     "_jupyter_labextension_paths",
+    "Server",
+    "Client",
     "PerspectiveError",
     "PerspectiveWidget",
-    "PerspectiveViewer",
-    "PerspectiveTornadoHandler",
     "ProxySession",
-    "Table",
-    "View",
-    "Server",
-    "Client",
 ]
 
+import functools
+
 from .perspective import (
-    Client as PySyncClient,
+    Client,
     PerspectiveError,
-    Table,
-    View,
     ProxySession,
-    PySyncServer,
+    PySyncServer as Server,
 )
 
-try:
-    from .widget import PerspectiveWidget
-except ImportError:
-    ...
-try:
-    from .viewer import PerspectiveViewer
-except ImportError:
-    ...
-
-# from .psp_cffi import ServerBase
-
-try:
-    from .handlers import PerspectiveTornadoHandler
-except ImportError:
-    ...
-
-
-def default_loop_cb(fn, *args, **kwargs):
-    return fn(*args, **kwargs)
-
-
-class Server(PySyncServer):
-    def set_threadpool_size(self, n_cpus):
-        pass
 
-    def new_local_client(self, loop_callback=default_loop_cb):
-        """Create a new `Client` instance bound to this in-process `Server`."""
-        return Client.from_server(self, loop_callback)
+GLOBAL_SERVER = Server()
+GLOBAL_CLIENT = GLOBAL_SERVER.new_local_client()
 
 
-class Client(PySyncClient):
-    def from_server(
-        server: Server,
-        loop_callback=default_loop_cb,
-    ):
-        """Create a new `Client` instance bound synchronously to an Python
-        instance of `PerspectiveServer`."""
+@functools.wraps(Client.table)
+def table(*args, **kwargs):
+    return GLOBAL_CLIENT.table(*args, **kwargs)
 
-        def handle_request(bytes):
-            session.handle_request(bytes)
-            loop_callback(lambda: session.poll())
 
-        def handle_response(bytes):
-            client.handle_response(bytes)
+@functools.wraps(Client.open_table)
+def open_table(*args, **kwargs):
+    return GLOBAL_CLIENT.table(*args, **kwargs)
 
-        def handle_close():
-            session.close()
 
-        session = server.new_session(handle_response)
-        client = Client(handle_request, handle_close)
-        return client
+@functools.wraps(Client.get_hosted_table_names)
+def get_hosted_table_names(*args, **kwargs):
+    return GLOBAL_CLIENT.get_hosted_table_names(*args, **kwargs)
 
 
 # Read by `jupyter labextension develop`

rust/perspective-python/perspective/handlers/__init__.py

@@ -9,18 +9,3 @@
 #  ┃ This file is part of the Perspective library, distributed under the terms ┃
 #  ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
 #  ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
-
-try:
-    from .aiohttp import PerspectiveAIOHTTPHandler  # noqa: F401
-except ImportError:
-    ...
-
-try:
-    from .starlette import PerspectiveStarletteHandler  # noqa: F401
-except ImportError:
-    ...
-
-try:
-    from .tornado import PerspectiveTornadoHandler  # noqa: F401
-except ImportError:
-    ...