From a6cd2aa0218002618914a1ca931e72286649618f Mon Sep 17 00:00:00 2001 From: Behnam Robatmili Date: Wed, 11 Oct 2023 11:00:26 -0700 Subject: [PATCH] Adding some example --- apis/python/src/tiledbsoma/IntIndexer.py | 2 +- apis/python/src/tiledbsoma/_fast_csr.py | 24 +++++++++---------- .../src/reindexer/examples/ann_data.py | 22 +++++++++++++++++ .../src/reindexer/examples/test_indexer.py | 14 +++++++++++ libtiledbsoma/src/reindexer/test_indexer.py | 14 +++++++++++ 5 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 libtiledbsoma/src/reindexer/examples/ann_data.py create mode 100644 libtiledbsoma/src/reindexer/examples/test_indexer.py create mode 100644 libtiledbsoma/src/reindexer/test_indexer.py diff --git a/apis/python/src/tiledbsoma/IntIndexer.py b/apis/python/src/tiledbsoma/IntIndexer.py index bc9d51127c..f31d9540bd 100644 --- a/apis/python/src/tiledbsoma/IntIndexer.py +++ b/apis/python/src/tiledbsoma/IntIndexer.py @@ -17,7 +17,7 @@ def map_locations(keys: np.ndarray[np.int64, Any]) -> clib.IntIndexer: ) compute_concurrency = 5 if context._tiledb_ctx: - compute_concurrency = context._tiledb_ctx.config()[ + compute_concurrency = int(int(context._tiledb_ctx.config()))/2[ "sm.compute_concurrency_level" ] thread_counts = int(compute_concurrency) diff --git a/apis/python/src/tiledbsoma/_fast_csr.py b/apis/python/src/tiledbsoma/_fast_csr.py index 8913f20dd1..11cad28663 100644 --- a/apis/python/src/tiledbsoma/_fast_csr.py +++ b/apis/python/src/tiledbsoma/_fast_csr.py @@ -45,9 +45,9 @@ class _CSRAccumulatorFinalResult(NamedTuple): Contains a sparse CSR consituent elements """ - data: npt.NDArray[np.number] # type: ignore[type-arg] - indptr: npt.NDArray[np.integer] # type: ignore[type-arg] - indices: npt.NDArray[np.integer] # type: ignore[type-arg] + data: npt.NDArray[np.number] # type: ignore + indptr: npt.NDArray[np.integer] # type: ignore + indices: npt.NDArray[np.integer] # type: ignore shape: Tuple[int, int] @@ -187,11 +187,11 @@ def _accum_row_length( @numba.jit(nopython=True, nogil=True) # type: ignore[attr-defined] def _copy_chunk_range( row_ind_chunk: npt.NDArray[np.signedinteger], # ignore-type - col_ind_chunk: npt.NDArray[np.signedinteger], - data_chunk: npt.NDArray[np.number], - data: npt.NDArray[np.number], - indices: npt.NDArray[np.signedinteger], - indptr: npt.NDArray[np.signedinteger], + col_ind_chunk: npt.NDArray[np.signedinteger], # type: ignore + data_chunk: npt.NDArray[np.number], # type: ignore + data: npt.NDArray[np.number], # type: ignore + indices: npt.NDArray[np.signedinteger], # type: ignore + indptr: npt.NDArray[np.signedinteger], # type: ignore row_rng_mask: int, row_rng_val: int, ) -> None: @@ -208,9 +208,9 @@ def _copy_chunk_range( @numba.jit(nopython=True, nogil=True) # type: ignore[attr-defined] def _copy_chunklist_range( chunk_list: numba.typed.List, - data: npt.NDArray[np.number], # type: ignore[attr-defined] - indices: npt.NDArray[np.signedinteger], # type: ignore[attr-defined] - indptr: npt.NDArray[np.signedinteger], # type: ignore[attr-defined] + data: npt.NDArray[np.number], # type: ignore + indices: npt.NDArray[np.signedinteger], # type: ignore + indptr: npt.NDArray[np.signedinteger], # type: ignore row_rng_mask_bits: int, job: int, ): @@ -284,7 +284,7 @@ def _read_csr( acc = _CSRAccumulator( obs_joinids=obs_joinids, var_joinids=var_joinids, pool=pool ) - for tbl in somacore._eager_iter.EagerIterator( + for tbl in somacore.query._eager_iter.EagerIterator( matrix.read((obs_joinids, var_joinids)).tables(), pool=pool, ): diff --git a/libtiledbsoma/src/reindexer/examples/ann_data.py b/libtiledbsoma/src/reindexer/examples/ann_data.py new file mode 100644 index 0000000000..f7a24efb5a --- /dev/null +++ b/libtiledbsoma/src/reindexer/examples/ann_data.py @@ -0,0 +1,22 @@ +import cellxgene_census +import tiledbsoma as soma +from time import perf_counter + +soma._query.ExperimentAxisQuery.USE_FAST_INDEXER = False + +census_S3_latest = dict(census_version="latest") +census_local_copy = dict(uri="/Users/brobatmili/projects/census_data/") + +def main(): + t1 = perf_counter() + with cellxgene_census.open_soma(**census_S3_latest, context=soma.SOMATileDBContext(use_optimized_reindexing=True)) as census: + with census["census_data"]["homo_sapiens"].axis_query( + measurement_name="RNA", obs_query=soma.AxisQuery( + value_filter="""tissue_general == 'eye'""") + + ) as query: + query.to_anndata(X_name="raw") + t2 = perf_counter() + print(f"End to end time {t2 - t1}") + +main() \ No newline at end of file diff --git a/libtiledbsoma/src/reindexer/examples/test_indexer.py b/libtiledbsoma/src/reindexer/examples/test_indexer.py new file mode 100644 index 0000000000..698927f9a5 --- /dev/null +++ b/libtiledbsoma/src/reindexer/examples/test_indexer.py @@ -0,0 +1,14 @@ +from tiledbsoma.IntIndexer import IntIndexer +import numpy as np + +# 1d array to list +keys = np.array([-1, 1, 2, 3, 4, 5]) +look_ups = np.array([-1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5]) + +def main(): + indexer = IntIndexer.map_locations(keys) + results = indexer.lookup(look_ups) + print(f"Lookup results {results}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/libtiledbsoma/src/reindexer/test_indexer.py b/libtiledbsoma/src/reindexer/test_indexer.py new file mode 100644 index 0000000000..698927f9a5 --- /dev/null +++ b/libtiledbsoma/src/reindexer/test_indexer.py @@ -0,0 +1,14 @@ +from tiledbsoma.IntIndexer import IntIndexer +import numpy as np + +# 1d array to list +keys = np.array([-1, 1, 2, 3, 4, 5]) +look_ups = np.array([-1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5, -1, 1, 2, 3, 4, 5]) + +def main(): + indexer = IntIndexer.map_locations(keys) + results = indexer.lookup(look_ups) + print(f"Lookup results {results}") + +if __name__ == "__main__": + main() \ No newline at end of file