From a760eced0f26d604d4d03830e2f20635ee54e38d Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 23 Jul 2023 11:48:10 -0400 Subject: [PATCH 1/3] Fix double-view creation when `restore()` called after `load()` within a microtask --- .prettierignore | 3 +- cpp/perspective/src/cpp/view.cpp | 24 ++++--- .../perspective/manager/manager_internal.py | 2 +- rust/perspective-viewer/src/rust/session.rs | 62 +++++++++++-------- 4 files changed, 56 insertions(+), 35 deletions(-) diff --git a/.prettierignore b/.prettierignore index 2145c5c7a9..5be77c1829 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ build/ node_modules/ target/ .emsdk/ -boost*/ \ No newline at end of file +boost*/ +py_modules \ No newline at end of file diff --git a/cpp/perspective/src/cpp/view.cpp b/cpp/perspective/src/cpp/view.cpp index 74b83a524f..9e9c718f40 100644 --- a/cpp/perspective/src/cpp/view.cpp +++ b/cpp/perspective/src/cpp/view.cpp @@ -1580,10 +1580,10 @@ View::to_columns(t_uindex start_row, t_uindex end_row, bool get_pkeys, bool get_ids, bool _leaves_only, t_uindex num_sides, bool _has_row_path, std::string nidx, t_uindex columns_length, t_uindex group_by_length) const { - + PSP_GIL_UNLOCK(); + PSP_READ_LOCK(get_lock()); auto slice = get_data(start_row, end_row, start_col, end_col); - auto col_names = slice->get_column_names(); - auto schema = m_ctx->get_schema(); + auto& col_names = slice->get_column_names(); rapidjson::StringBuffer s; rapidjson::Writer writer(s); @@ -1628,9 +1628,12 @@ View::to_columns(t_uindex start_row, t_uindex end_row, bool get_pkeys, bool get_ids, bool _leaves_only, t_uindex num_sides, bool _has_row_path, std::string nidx, t_uindex columns_length, t_uindex group_by_length) const { + PSP_GIL_UNLOCK(); + PSP_READ_LOCK(get_lock()); auto slice = get_data(start_row, end_row, start_col, end_col); - auto col_names = slice->get_column_names(); - auto schema = m_ctx->get_schema(); + const std::vector>& col_names + = slice->get_column_names(); + rapidjson::StringBuffer s; rapidjson::Writer writer(s); writer.StartObject(); @@ -1671,8 +1674,11 @@ View::to_columns(t_uindex start_row, t_uindex end_row, bool get_pkeys, bool get_ids, bool leaves_only, t_uindex num_sides, bool has_row_path, std::string nidx, t_uindex columns_length, t_uindex group_by_length) const { + PSP_GIL_UNLOCK(); + PSP_READ_LOCK(get_lock()); + auto slice = get_data(start_row, end_row, start_col, end_col); - auto col_names = slice->get_column_names(); + const auto& col_names = slice->get_column_names(); rapidjson::StringBuffer s; rapidjson::Writer writer(s); writer.StartObject(); @@ -1721,8 +1727,10 @@ View::to_columns(t_uindex start_row, t_uindex end_row, bool get_pkeys, bool get_ids, bool leaves_only, t_uindex num_sides, bool has_row_path, std::string nidx, t_uindex columns_length, t_uindex group_by_length) const { - auto slice = get_data(start_row, end_row, start_col, end_col); - auto col_names = slice->get_column_names(); + PSP_GIL_UNLOCK(); + PSP_READ_LOCK(get_lock()); + const auto slice = get_data(start_row, end_row, start_col, end_col); + const auto& col_names = slice->get_column_names(); rapidjson::StringBuffer s; rapidjson::Writer writer(s); writer.StartObject(); diff --git a/python/perspective/perspective/manager/manager_internal.py b/python/perspective/perspective/manager/manager_internal.py index 9edbe58743..ece4889272 100644 --- a/python/perspective/perspective/manager/manager_internal.py +++ b/python/perspective/perspective/manager/manager_internal.py @@ -355,7 +355,7 @@ def callback(self, *args, **kwargs): else: msg = self._make_message(id, None) - if len(args) > 1 and type(args[1]) == bytes: + if len(args) > 1 and type(args[1]) is bytes: self._process_bytes(args[1], msg, post_callback) else: post_callback(self._message_to_json(msg["id"], msg)) diff --git a/rust/perspective-viewer/src/rust/session.rs b/rust/perspective-viewer/src/rust/session.rs index 5279ad5d7f..60f226a31c 100644 --- a/rust/perspective-viewer/src/rust/session.rs +++ b/rust/perspective-viewer/src/rust/session.rs @@ -66,6 +66,7 @@ pub struct SessionData { config: ViewConfig, view_sub: Option, stats: Option, + is_clean: bool, } impl Deref for Session { @@ -107,6 +108,7 @@ impl Session { /// # Arguments /// - `keep_expressions` Whether to reset the `expressions` property. pub fn reset(&self, reset_expressions: bool) { + self.borrow_mut().is_clean = false; self.borrow_mut().view_sub = None; self.borrow_mut().config.reset(reset_expressions); } @@ -306,8 +308,9 @@ impl Session { /// Update the config, setting the `columns` property to the plugin defaults /// if provided. pub fn update_view_config(&self, config_update: ViewConfigUpdate) { - self.borrow_mut().view_sub = None; if self.borrow_mut().config.apply_update(config_update) { + self.borrow_mut().view_sub = None; + self.0.borrow_mut().is_clean = false; self.view_config_changed.emit_all(()); } } @@ -452,6 +455,12 @@ impl Session { self.borrow_mut().config = config; Ok(()) } + + fn reset_clean(&self) -> bool { + let mut is_clean = true; + std::mem::swap(&mut is_clean, &mut self.0.borrow_mut().is_clean); + is_clean + } } /// A newtype wrapper which only provides `create_view()` @@ -463,31 +472,34 @@ impl<'a> ValidSession<'a> { /// the original `&Session`. pub async fn create_view(&self) -> Result<&'a Session, ApiError> { let js_config = self.0.borrow().config.as_jsvalue()?; - let table = self - .0 - .borrow() - .table - .clone() - .ok_or("`restore()` called before `load()`")?; - - let view = table.view(&js_config).await?; - let view_schema = view.schema().await?; - self.0.metadata_mut().update_view_schema(&view_schema)?; - - let on_stats = Callback::from({ - let this = self.0.clone(); - move |stats| this.update_stats(stats) - }); - - let sub = { - let config = self.0.borrow().config.clone(); - let on_update = self.0.table_updated.callback(); - ViewSubscription::new(view, config, on_stats, on_update) - }; + if !self.0.reset_clean() { + let table = self + .0 + .borrow() + .table + .clone() + .ok_or("`restore()` called before `load()`")?; + + let view = table.view(&js_config).await?; + let view_schema = view.schema().await?; + self.0.metadata_mut().update_view_schema(&view_schema)?; + + let on_stats = Callback::from({ + let this = self.0.clone(); + move |stats| this.update_stats(stats) + }); + + let sub = { + let config = self.0.borrow().config.clone(); + let on_update = self.0.table_updated.callback(); + ViewSubscription::new(view, config, on_stats, on_update) + }; + + // self.0.borrow_mut().metadata.as_mut().unwrap().view_schema = + // Some(view_schema); + self.0.borrow_mut().view_sub = Some(sub); + } - // self.0.borrow_mut().metadata.as_mut().unwrap().view_schema = - // Some(view_schema); - self.0.borrow_mut().view_sub = Some(sub); Ok(self.0) } } From d557cef047d8056dd786ffce88c98d8a36f9a391 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 2 Aug 2023 00:35:55 -0400 Subject: [PATCH 2/3] Fix double expression evaluation on `view()` --- .../src/cpp/context_grouped_pkey.cpp | 8 --- cpp/perspective/src/cpp/context_one.cpp | 8 --- cpp/perspective/src/cpp/context_two.cpp | 8 --- cpp/perspective/src/cpp/context_zero.cpp | 8 --- cpp/perspective/src/cpp/expression_tables.cpp | 13 +++++ cpp/perspective/src/cpp/gnode.cpp | 55 +++++++++++++++---- cpp/perspective/src/cpp/gnode_state.cpp | 21 ++++--- .../perspective/context_common_decls.h | 1 - .../include/perspective/expression_tables.h | 2 + .../src/include/perspective/gnode_state.h | 2 + 10 files changed, 72 insertions(+), 54 deletions(-) diff --git a/cpp/perspective/src/cpp/context_grouped_pkey.cpp b/cpp/perspective/src/cpp/context_grouped_pkey.cpp index 46400da521..486f9dbed0 100644 --- a/cpp/perspective/src/cpp/context_grouped_pkey.cpp +++ b/cpp/perspective/src/cpp/context_grouped_pkey.cpp @@ -684,7 +684,6 @@ t_ctx_grouped_pkey::get_column_dtype(t_uindex idx) const { void t_ctx_grouped_pkey::compute_expressions(std::shared_ptr master, - std::shared_ptr flattened, t_expression_vocab& expression_vocab, t_regex_mapping& regex_mapping) { // Clear the transitional expression tables on the context so they are // ready for the next update. @@ -693,10 +692,6 @@ t_ctx_grouped_pkey::compute_expressions(std::shared_ptr master, std::shared_ptr master_expression_table = m_expression_tables->m_master; - t_uindex flattened_num_rows = flattened->size(); - m_expression_tables->reserve_transitional_table_size(flattened_num_rows); - m_expression_tables->set_transitional_table_size(flattened_num_rows); - // Set the master table to the right size. t_uindex num_rows = master->size(); master_expression_table->reserve(num_rows); @@ -707,9 +702,6 @@ t_ctx_grouped_pkey::compute_expressions(std::shared_ptr master, // Compute the expressions on the master table. expr->compute( master, master_expression_table, expression_vocab, regex_mapping); - - expr->compute(flattened, m_expression_tables->m_flattened, - expression_vocab, regex_mapping); } } diff --git a/cpp/perspective/src/cpp/context_one.cpp b/cpp/perspective/src/cpp/context_one.cpp index 0137c45d3c..7a01a7d6d0 100644 --- a/cpp/perspective/src/cpp/context_one.cpp +++ b/cpp/perspective/src/cpp/context_one.cpp @@ -612,7 +612,6 @@ t_ctx1::get_trav_depth(t_index idx) const { void t_ctx1::compute_expressions(std::shared_ptr master, - std::shared_ptr flattened, t_expression_vocab& expression_vocab, t_regex_mapping& regex_mapping) { // Clear the transitional expression tables on the context so they are // ready for the next update. @@ -621,10 +620,6 @@ t_ctx1::compute_expressions(std::shared_ptr master, std::shared_ptr master_expression_table = m_expression_tables->m_master; - t_uindex flattened_num_rows = flattened->size(); - m_expression_tables->reserve_transitional_table_size(flattened_num_rows); - m_expression_tables->set_transitional_table_size(flattened_num_rows); - // Set the master table to the right size. t_uindex num_rows = master->size(); master_expression_table->reserve(num_rows); @@ -635,9 +630,6 @@ t_ctx1::compute_expressions(std::shared_ptr master, // Compute the expressions on the master table. expr->compute( master, master_expression_table, expression_vocab, regex_mapping); - - expr->compute(flattened, m_expression_tables->m_flattened, - expression_vocab, regex_mapping); } } diff --git a/cpp/perspective/src/cpp/context_two.cpp b/cpp/perspective/src/cpp/context_two.cpp index 66d52f3b4b..877d404eaf 100644 --- a/cpp/perspective/src/cpp/context_two.cpp +++ b/cpp/perspective/src/cpp/context_two.cpp @@ -1054,7 +1054,6 @@ t_ctx2::get_column_dtype(t_uindex idx) const { void t_ctx2::compute_expressions(std::shared_ptr master, - std::shared_ptr flattened, t_expression_vocab& expression_vocab, t_regex_mapping& regex_mapping) { // Clear the transitional expression tables on the context so they are // ready for the next update. @@ -1063,10 +1062,6 @@ t_ctx2::compute_expressions(std::shared_ptr master, std::shared_ptr master_expression_table = m_expression_tables->m_master; - t_uindex flattened_num_rows = flattened->size(); - m_expression_tables->reserve_transitional_table_size(flattened_num_rows); - m_expression_tables->set_transitional_table_size(flattened_num_rows); - // Set the master table to the right size. t_uindex num_rows = master->size(); master_expression_table->reserve(num_rows); @@ -1077,9 +1072,6 @@ t_ctx2::compute_expressions(std::shared_ptr master, // Compute the expressions on the master table. expr->compute( master, master_expression_table, expression_vocab, regex_mapping); - - expr->compute(flattened, m_expression_tables->m_flattened, - expression_vocab, regex_mapping); } } diff --git a/cpp/perspective/src/cpp/context_zero.cpp b/cpp/perspective/src/cpp/context_zero.cpp index ee5257094b..a0dc9acd8f 100644 --- a/cpp/perspective/src/cpp/context_zero.cpp +++ b/cpp/perspective/src/cpp/context_zero.cpp @@ -620,7 +620,6 @@ t_ctx0::get_step_delta(t_index bidx, t_index eidx) { void t_ctx0::compute_expressions(std::shared_ptr master, - std::shared_ptr flattened, t_expression_vocab& expression_vocab, t_regex_mapping& regex_mapping) { // Clear the transitional expression tables on the context so they are // ready for the next update. @@ -629,10 +628,6 @@ t_ctx0::compute_expressions(std::shared_ptr master, std::shared_ptr master_expression_table = m_expression_tables->m_master; - t_uindex flattened_num_rows = flattened->size(); - m_expression_tables->reserve_transitional_table_size(flattened_num_rows); - m_expression_tables->set_transitional_table_size(flattened_num_rows); - // Set the master table to the right size. t_uindex num_rows = master->size(); master_expression_table->reserve(num_rows); @@ -643,9 +638,6 @@ t_ctx0::compute_expressions(std::shared_ptr master, // Compute the expressions on the master table. expr->compute( master, master_expression_table, expression_vocab, regex_mapping); - - expr->compute(flattened, m_expression_tables->m_flattened, - expression_vocab, regex_mapping); } } diff --git a/cpp/perspective/src/cpp/expression_tables.cpp b/cpp/perspective/src/cpp/expression_tables.cpp index 531336b353..ec489e37a1 100644 --- a/cpp/perspective/src/cpp/expression_tables.cpp +++ b/cpp/perspective/src/cpp/expression_tables.cpp @@ -51,6 +51,19 @@ t_expression_tables::get_table() const { return m_master.get(); } +void +t_expression_tables::set_flattened(std::shared_ptr flattened) { + t_uindex flattened_num_rows = flattened->size(); + reserve_transitional_table_size(flattened_num_rows); + set_transitional_table_size(flattened_num_rows); + const t_schema& schema = m_flattened->get_schema(); + const std::vector& column_names = schema.m_columns; + for (const auto& colname : column_names) { + m_flattened->set_column( + colname, flattened->get_column(colname)->clone()); + } +} + void t_expression_tables::calculate_transitions( std::shared_ptr existed) { diff --git a/cpp/perspective/src/cpp/gnode.cpp b/cpp/perspective/src/cpp/gnode.cpp index 2650a55282..131bb6b1c1 100644 --- a/cpp/perspective/src/cpp/gnode.cpp +++ b/cpp/perspective/src/cpp/gnode.cpp @@ -871,8 +871,13 @@ t_gnode::_register_context( ctx->reset(); if (should_update) { - ctx->compute_expressions(m_gstate->get_table(), pkeyed_table, + ctx->compute_expressions(m_gstate->get_table(), expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); + update_context_from_state(ctx, name, pkeyed_table); } } break; @@ -881,8 +886,12 @@ t_gnode::_register_context( t_ctx1* ctx = static_cast(ptr_); ctx->reset(); if (should_update) { - ctx->compute_expressions(m_gstate->get_table(), pkeyed_table, + ctx->compute_expressions(m_gstate->get_table(), expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); update_context_from_state(ctx, name, pkeyed_table); } @@ -892,8 +901,13 @@ t_gnode::_register_context( t_ctx0* ctx = static_cast(ptr_); ctx->reset(); if (should_update) { - ctx->compute_expressions(m_gstate->get_table(), pkeyed_table, + ctx->compute_expressions(m_gstate->get_table(), expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); + update_context_from_state(ctx, name, pkeyed_table); } } break; @@ -913,8 +927,13 @@ t_gnode::_register_context( ctx->reset(); if (should_update) { - ctx->compute_expressions(m_gstate->get_table(), pkeyed_table, + ctx->compute_expressions(m_gstate->get_table(), expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); + update_context_from_state( ctx, name, pkeyed_table); } @@ -999,27 +1018,39 @@ t_gnode::_compute_expressions(std::shared_ptr flattened_masked) { case TWO_SIDED_CONTEXT: { t_ctx2* ctx = static_cast(ctxh.m_ctx); ctx->compute_expressions(m_gstate->get_table(), - flattened_masked, expression_vocab, - expression_regex_mapping); + expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); } break; case ONE_SIDED_CONTEXT: { t_ctx1* ctx = static_cast(ctxh.m_ctx); ctx->compute_expressions(m_gstate->get_table(), - flattened_masked, expression_vocab, - expression_regex_mapping); + expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); } break; case ZERO_SIDED_CONTEXT: { t_ctx0* ctx = static_cast(ctxh.m_ctx); ctx->compute_expressions(m_gstate->get_table(), - flattened_masked, expression_vocab, - expression_regex_mapping); + expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); } break; case GROUPED_PKEY_CONTEXT: { t_ctx_grouped_pkey* ctx = static_cast(ctxh.m_ctx); ctx->compute_expressions(m_gstate->get_table(), - flattened_masked, expression_vocab, - expression_regex_mapping); + expression_vocab, expression_regex_mapping); + ctx->get_expression_tables()->set_flattened( + m_gstate->get_pkeyed_table( + ctx->get_expression_tables()->m_master->get_schema(), + ctx->get_expression_tables()->m_master)); } break; case UNIT_CONTEXT: break; diff --git a/cpp/perspective/src/cpp/gnode_state.cpp b/cpp/perspective/src/cpp/gnode_state.cpp index 55920d46e2..4c82616c9c 100644 --- a/cpp/perspective/src/cpp/gnode_state.cpp +++ b/cpp/perspective/src/cpp/gnode_state.cpp @@ -547,10 +547,16 @@ t_gstate::get_pkey_dtype() const { std::shared_ptr t_gstate::get_pkeyed_table() const { + return get_pkeyed_table(m_input_schema, m_table); +} + +std::shared_ptr +t_gstate::get_pkeyed_table( + const t_schema& schema, const std::shared_ptr table) const { // If there are no removes, just return the gstate table. Removes would // cause m_mapping to be smaller than m_table. - if (m_mapping.size() == m_table->size()) - return m_table; + if (m_mapping.size() == table->size()) + return table; // Otherwise mask out the removed rows and return the table. auto mask = get_cpp_mask(); @@ -558,22 +564,19 @@ t_gstate::get_pkeyed_table() const { // count = total number of rows - number of removed rows t_uindex table_size = mask.count(); - const auto& schema_columns = m_input_schema.m_columns; + const auto& schema_columns = schema.m_columns; t_uindex num_columns = schema_columns.size(); - // Clone from the gstate master table - const std::shared_ptr& master_table = m_table; - std::shared_ptr rval - = std::make_shared(m_input_schema, table_size); + = std::make_shared(schema, table_size); rval->init(); rval->set_size(table_size); parallel_for(int(num_columns), - [&schema_columns, rval, master_table, &mask](int colidx) { + [&schema_columns, rval, table, &mask](int colidx) { const std::string& colname = schema_columns[colidx]; rval->set_column( - colname, master_table->get_const_column(colname)->clone(mask)); + colname, table->get_const_column(colname)->clone(mask)); } ); diff --git a/cpp/perspective/src/include/perspective/context_common_decls.h b/cpp/perspective/src/include/perspective/context_common_decls.h index 83d9a33d2d..c8224e5970 100644 --- a/cpp/perspective/src/include/perspective/context_common_decls.h +++ b/cpp/perspective/src/include/perspective/context_common_decls.h @@ -97,7 +97,6 @@ std::shared_ptr get_expression_tables() const; // Given shared pointers to data tables from the gnode, use them to // compute the results of expression columns. void compute_expressions(std::shared_ptr master, - std::shared_ptr flattened_masked, t_expression_vocab& expression_vocab, t_regex_mapping& regex_mapping); void compute_expressions(std::shared_ptr master, diff --git a/cpp/perspective/src/include/perspective/expression_tables.h b/cpp/perspective/src/include/perspective/expression_tables.h index 2d09bb8317..8ab5060615 100644 --- a/cpp/perspective/src/include/perspective/expression_tables.h +++ b/cpp/perspective/src/include/perspective/expression_tables.h @@ -50,6 +50,8 @@ struct t_expression_tables { // Calculate the `t_transitions` value for each row. void calculate_transitions(std::shared_ptr existed); + void set_flattened(std::shared_ptr flattened); + void reset(); t_data_table* get_table() const; diff --git a/cpp/perspective/src/include/perspective/gnode_state.h b/cpp/perspective/src/include/perspective/gnode_state.h index c450340774..2435e865c3 100644 --- a/cpp/perspective/src/include/perspective/gnode_state.h +++ b/cpp/perspective/src/include/perspective/gnode_state.h @@ -199,6 +199,8 @@ class PERSPECTIVE_EXPORT t_gstate { // Getters std::shared_ptr get_table() const; std::shared_ptr get_pkeyed_table() const; + std::shared_ptr get_pkeyed_table(const t_schema& schema, + const std::shared_ptr table) const; const t_schema& get_input_schema() const; const t_schema& get_output_schema() const; From 1a2cb3adb1e81f844388384c39544f20ff1108f6 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 2 Aug 2023 00:36:21 -0400 Subject: [PATCH 3/3] Pin python dependencies --- .github/workflows/build.yml | 16 +- python/perspective/requirements-dev.txt | 157 ++++++++++++++++++ .../perspective-scripts/_requires_python.mjs | 24 +-- tools/perspective-scripts/_wheel_python.mjs | 2 +- tools/perspective-scripts/build_python.mjs | 4 +- 5 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 python/perspective/requirements-dev.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ac40c8c5d..4aef5120c6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1359,10 +1359,10 @@ jobs: # Install artifact - name: Install wheel (system) - run: python -m pip install -U *manylinux2014*.whl + run: python -m pip install -r python/perspective/requirements-dev.txt *manylinux2014*.whl - name: Install wheel (local) - run: python -m pip install -U *manylinux2014*.whl --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt *manylinux2014*.whl --target python/perspective - name: Check Installed labextensions run: jupyter labextension list @@ -1541,21 +1541,21 @@ jobs: # Install artifact in-place so tests work as-is - name: Install wheel (Linux) - run: python -m pip install -U *manylinux2014*.whl --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt *manylinux2014*.whl --target python/perspective if: ${{ runner.os == 'Linux' }} # Note, on mac we must install the x86 wheel, the arm64 wheel # would need an arm machine to test - name: Install wheel (OSX) - run: python -m pip install -U *x86*.whl --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt *x86*.whl --target python/perspective if: ${{ runner.os == 'macOS' && matrix.python-version != '3.11' }} - name: Install wheel (OSX 3.11+) - run: python -m pip install -U *universal*.whl --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt *universal*.whl --target python/perspective if: ${{ runner.os == 'macOS' && matrix.python-version == '3.11' }} - name: Install wheel (windows) - run: python -m pip install -U (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target python/perspective if: ${{ runner.os == 'Windows' }} # Run tests @@ -1683,7 +1683,7 @@ jobs: # Install sdist - name: Install sdist - run: python -m pip install perspective*.tar.gz + run: python -m pip install -r python/perspective/requirements-dev.txt perspective*.tar.gz # Test sdist - name: Run tests against from-scratch sdist build @@ -1801,7 +1801,7 @@ jobs: # Install artifact in-place so tests work as-is - name: Install wheel (Linux) - run: python -m pip install -U *manylinux2014*.whl --target python/perspective + run: python -m pip install -r python/perspective/requirements-dev.txt *manylinux2014*.whl --target python/perspective if: ${{ runner.os == 'Linux' }} - name: Run Benchmark diff --git a/python/perspective/requirements-dev.txt b/python/perspective/requirements-dev.txt new file mode 100644 index 0000000000..01dee2553a --- /dev/null +++ b/python/perspective/requirements-dev.txt @@ -0,0 +1,157 @@ +aiofiles==22.1.0 +aiohttp==3.8.5 +aiosignal==1.3.1 +aiosqlite==0.19.0 +alabaster==0.7.13 +annotated-types==0.5.0 +anyio==3.7.1 +appnope==0.1.3 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +arrow==1.2.3 +asttokens==2.2.1 +async-timeout==4.0.2 +attrs==23.1.0 +Babel==2.12.1 +backcall==0.2.0 +beautifulsoup4==4.12.2 +black==23.1.0 +bleach==6.0.0 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.2.0 +click==8.1.6 +comm==0.1.3 +coverage==7.2.7 +debugpy==1.6.7 +decorator==5.1.1 +defusedxml==0.7.1 +deprecation==2.1.0 +docutils==0.20.1 +entrypoints==0.4 +executing==1.2.0 +Faker==19.2.0 +fastapi==0.100.1 +fastjsonschema==2.18.0 +flake8==6.1.0 +flake8-black==0.3.6 +fqdn==1.5.1 +frozenlist==1.4.0 +future==0.18.3 +h11==0.14.0 +html5lib==1.1 +httpcore==0.17.3 +httpx==0.24.1 +idna==3.4 +imagesize==1.4.1 +iniconfig==2.0.0 +ipykernel==6.25.0 +ipython==8.14.0 +ipython-genutils==0.2.0 +ipywidgets==8.0.7 +isoduration==20.11.0 +jedi==0.19.0 +Jinja2==3.1.2 +json5==0.9.14 +jsonpointer==2.4 +jsonschema==4.18.4 +jsonschema-specifications==2023.7.1 +jupyter-events==0.6.3 +jupyter-ydoc==0.2.5 +jupyter_client==7.4.9 +jupyter_core==5.3.1 +jupyter_packaging==0.12.3 +jupyter_server==2.7.0 +jupyter_server_fileid==0.9.0 +jupyter_server_terminals==0.4.4 +jupyter_server_ydoc==0.8.0 +jupyterlab==3.6.5 +jupyterlab-pygments==0.2.2 +jupyterlab-widgets==3.0.8 +jupyterlab_server==2.24.0 +MarkupSafe==2.1.3 +matplotlib-inline==0.1.6 +mccabe==0.7.0 +mistune==3.0.1 +multidict==6.0.4 +mypy-extensions==1.0.0 +nbclassic==1.0.0 +nbclient==0.8.0 +nbconvert==7.7.3 +nbformat==5.9.1 +nest-asyncio==1.5.7 +notebook==6.5.5 +notebook_shim==0.2.3 +numpy==1.25.1 +overrides==7.3.1 +packaging==23.1 +pandas==2.0.3 +pandocfilters==1.5.0 +parso==0.8.3 +pathspec==0.11.2 +pexpect==4.8.0 +pickleshare==0.7.5 +platformdirs==3.10.0 +pluggy==1.2.0 +prometheus-client==0.17.1 +prompt-toolkit==3.0.39 +psutil==5.9.5 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==12.0.1 +pybind11==2.11.1 +pycodestyle==2.11.0 +pycparser==2.21 +pydantic==2.1.1 +pydantic_core==2.4.0 +pyflakes==3.1.0 +Pygments==2.15.1 +pytest==7.4.0 +pytest-aiohttp==1.0.4 +pytest-asyncio==0.21.1 +pytest-cov==4.1.0 +pytest-tornado==0.8.1 +pytest_check_links==0.9.0 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pytz==2023.3 +PyYAML==6.0.1 +pyzmq==24.0.1 +referencing==0.30.0 +requests==2.31.0 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rpds-py==0.9.2 +Send2Trash==1.8.2 +six==1.16.0 +sniffio==1.3.0 +snowballstemmer==2.2.0 +soupsieve==2.4.1 +Sphinx==7.1.1 +sphinx-markdown-builder==0.6.4 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +stack-data==0.6.2 +starlette==0.27.0 +tabulate==0.9.0 +terminado==0.17.1 +tinycss2==1.2.1 +tomlkit==0.12.1 +tornado==6.3.2 +traitlets==5.9.0 +typing_extensions==4.7.1 +tzdata==2023.3 +uri-template==1.3.0 +urllib3==2.0.4 +wcwidth==0.2.6 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.6.1 +widgetsnbextension==4.0.8 +y-py==0.6.0 +yarl==1.9.2 +ypy-websocket==0.8.4 \ No newline at end of file diff --git a/tools/perspective-scripts/_requires_python.mjs b/tools/perspective-scripts/_requires_python.mjs index 562c57a4f3..41a6cbf648 100644 --- a/tools/perspective-scripts/_requires_python.mjs +++ b/tools/perspective-scripts/_requires_python.mjs @@ -18,17 +18,21 @@ import { let PYTHON = sh(python_version()); -const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`; +if (process.env.PSP_OLD_SHITTY_INSTALL_METHOD) { + const requires_script = `import distutils.core; setup = distutils.core.run_setup('python/perspective/setup.py'); print(' '.join(['"' + requirement + '"' for requirement in setup.extras_require['dev']]))`; -// copy build/config files into python folder -copy_files_to_python_folder(); + // copy build/config files into python folder + copy_files_to_python_folder(); -// install build meta deps, this is a necessary evil to keep the setup.py clean -sh`${PYTHON} -m pip install -U jupyter_packaging==0.12.3`.runSync(); -const requirements = await sh`${PYTHON} -c ${requires_script}`.execSync(); -if (requirements.trim().length > 0) { - console.log(`Installing: ${requirements}`); - sh`${PYTHON} -m pip install -U ${sh(requirements)}`.log().runSync(); + // install build meta deps, this is a necessary evil to keep the setup.py clean + sh`${PYTHON} -m pip install jupyter_packaging==0.12.3`.runSync(); + const requirements = await sh`${PYTHON} -c ${requires_script}`.execSync(); + if (requirements.trim().length > 0) { + console.log(`Installing: ${requirements}`); + sh`${PYTHON} -m pip install -U ${sh(requirements)}`.log().runSync(); + } else { + console.log("Nothing to install"); + } } else { - console.log("Nothing to install"); + sh`${PYTHON} -m pip install -r python/perspective/requirements-dev.txt`.runSync(); } diff --git a/tools/perspective-scripts/_wheel_python.mjs b/tools/perspective-scripts/_wheel_python.mjs index 1861347c8d..bd3e31ba5c 100644 --- a/tools/perspective-scripts/_wheel_python.mjs +++ b/tools/perspective-scripts/_wheel_python.mjs @@ -59,7 +59,7 @@ if (IS_DOCKER) { // These are system deps that may only be in place from pep-517/518 so // lets reinstall them to be sure - cmd.sh`${PYTHON} -m pip install -U 'numpy>=1.13.1' jupyter_packaging wheel twine auditwheel`; + cmd.sh`${PYTHON} -m pip install 'numpy>=1.13.1' jupyter_packaging wheel twine auditwheel`; // remove the build folder so we completely rebuild (and pick up the // libs we just installed above, since this build method won't use diff --git a/tools/perspective-scripts/build_python.mjs b/tools/perspective-scripts/build_python.mjs index 9593bad5ef..c7902ce749 100644 --- a/tools/perspective-scripts/build_python.mjs +++ b/tools/perspective-scripts/build_python.mjs @@ -47,9 +47,9 @@ if (SETUP_ONLY) { let cmd; if (IS_CI) { - cmd = sh`${PYTHON} -m pip install -e .[dev] --no-clean`; + cmd = sh`${PYTHON} -m pip install -r ./requirements-dev.txt -e .[dev] --no-clean`; } else if (IS_INSTALL) { - cmd = sh`${PYTHON} -m pip install .`; + cmd = sh`${PYTHON} -m pip install -r ./requirements-dev.txt .`; } else if (IS_PYODIDE) { cmd = sh`pyodide build . --exports=pyinit`; } else {