Merge branch 'main' into fix_unit_tests

.gitignore

@@ -106,7 +106,7 @@ ipython_config.py
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 #   install all needed dependencies.
-#Pipfile.lock
+Pipfile.lock
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
@@ -146,5 +146,9 @@ dmypy.json
 .pyre/
 
 report-*
-Pipfile.lock
+
+# poetry lock file
 poetry.lock
+
+*.zip
+.DS_Store

.pre-commit-config.yaml

@@ -36,18 +36,3 @@ repos:
         exclude: ^examples\/|.*\.ipynb
         language: python
         types: [text]
--   repo: https://github.com/python-poetry/poetry
-    rev: ''  # add version here
-    hooks:
-    -   id: poetry-check
-    -   id: poetry-lock
-    -   id: poetry-export
-        # TODO: check is it a single argument or multiple arguments
-        args: ["-f", "requirements.txt", "-o", "requirements.txt"]
-        args: ["--dev", "-f", "requirements-dev.txt", "-o", "requirements-dev.txt"]
-        args: ["--tut"-f", "requirements-tut.txt", "-o", "requirements-tut.txt"]
-
-        verbose: true
-
-
-

MANIFEST.in

@@ -4,33 +4,26 @@ include CITATION
 include LICENSE
 include Makefile
 include changelog.txt
-include dat/*.csv
-include dat/*.db
-include dat/*.ipynb
-include dat/*.json
-include dat/*.jsonld
-include dat/*.parquet
-include dat/*.ttl
-include dat/*.yml
-include dat/food_com/*.csv
-include dat/psl/*.json
-include dat/psl/*.txt
-include dat/titanic/*.csv
 include docker-compose.yml
 include environment.yml
-include examples/*.ipynb
-include examples/graph_algebra/*.ipynb
 include kglab/*.py
 include kglab/query/*.py
 include pylintrc
-include pyproject.toml
 include requirements-dev.txt
 include requirements-tut.txt
 include requirements.txt
 include sample.py
 include setup.py
-exclude bin/*
-exclude docker/*
-exclude docs/*
-exclude tests/*
-exclude wip/*
+prune dat
+prune bin
+prune docker
+prune docs
+prune scripts
+prune tests
+prune wip
+prune .ipynb_checkpoints
+prune examples
+exclude codecov.yml
+exclude lgtm.yml
+exclude meta.yaml
+exclude mkdocs.yml

Pipfile

@@ -63,4 +63,4 @@ types-requests = ">=2.27"
 xmltodict = ">=0.12"
 
 [requires]
-python_version = "3.7"
+python_version = "3.7"

bin/push_pypi.sh

@@ -3,6 +3,6 @@
 ## debugging the uploaded README:
 # pandoc README.md --from markdown --to rst -s -o README.rst
 
-rm -rf dist
-python setup.py sdist bdist_wheel
+rm -rf dist build kglab.egg-info
+python setup.py sdist bdist_wheel -v
 twine upload --verbose dist/*

changelog.txt

@@ -1,11 +1,13 @@
 # `kglab` changelog
 
-## 0.6.4
+## 0.6.6
 
 2022-11-23
 
+  * something is not quite right about source distributions... kudos to <https://stackoverflow.com/questions/608855/excluding-a-top-level-directory-from-a-setuptools-package>
   * update `setuptools` to workaround potential vulnerabilities
   * create `pyproject.toml` to support upcoming `pip` release
+  * remove tests and other unnecessary files from source distribution
 
 
 ## 0.6.1

docs/ack.md

@@ -11,15 +11,15 @@ and to our contributors:
 [@louisguitton](https://github.com/louisguitton),
 [@tomaarsen](https://github.com/tomaarsen),
 [@Mec-iS](https://github.com/Mec-iS),
-[@cutterkom](https://github.com/cutterkom),
-[@RishiKumarRay](https://github.com/RishiKumarRay),
+[@jake-aft](https://github.com/jake-aft),
 [@Tpt](https://github.com/Tpt),
 [@ArenasGuerreroJulian](https://github.com/ArenasGuerreroJulian),
 [@fils](https://github.com/fils),
+[@cutterkom](https://github.com/cutterkom),
+[@RishiKumarRay](https://github.com/RishiKumarRay),
 [@gauravjaglan](https://github.com/gauravjaglan),
 [@pebbie](https://github.com/pebbie),
 [@CatChenal](https://github.com/CatChenal),
-[@jake-aft](https://github.com/jake-aft),
 [@dmoore247](https://github.com/dmoore247);
 plus general support from [Derwen, Inc.](https://derwen.ai/);
 the [Knowledge Graph Conference](https://www.knowledgegraph.tech/)
@@ -109,7 +109,7 @@ Source code for **kglab** plus its logo, documentation, and examples
 have an [MIT license](https://spdx.org/licenses/MIT.html) which is
 succinct and simplifies use in commercial applications.
 
-All materials herein are Copyright © 2020-2022 Derwen, Inc.
+All materials herein are Copyright © 2020-2023 Derwen, Inc.
 
 [![logo for Derwen, Inc.](https://derwen.ai/static/block_logo.png)](https://derwen.ai/)
 

kglab/algebra.py

@@ -78,3 +78,25 @@ def to_scipy_sparse (self):
         """
         self.check_attributes()
         return nx.to_scipy_sparse_array(self.nx_graph)
+
+    def to_zarr (self, func: str) -> "zzarr.core.Array":  # type: ignore
+        """
+Return value of a function in `kglab.algebra` in [Zarr format](https://pypi.org/project/zarr/).
+
+       Args:
+func str: a name of one functions in `kglab.algebra`: `to_undirect`, `to_adjacency`, `to_incidence`, `to_laplacian`
+
+       returns:
+zzarr.core.Array: values of requested `func`
+        """
+        self.check_attributes()  # type: ignore
+        try:
+            import zarr  # type: ignore
+        except ImportError:
+            raise ImportError("To use Zarr you need to install kglab with the required extra package: pip install kglab[with-zarr]")
+
+        data = getattr(self, func)()
+        array = zarr.create(data.shape, chunks=True)
+        array[:] = data
+
+        return array

kglab/version.py

@@ -11,7 +11,7 @@
 
 
 MIN_PY_VERSION: typing.Tuple = (3, 7,)
-__version__: str = "0.6.4"
+__version__: str = "0.6.6"
 
 
 def _versify (

pyproject.toml

@@ -1,10 +1,3 @@
-[build-system]
-build-backend = "setuptools.build_meta"
-requires = [
-    "setuptools",
-]
-
-[project]
 name = "kglab"
 version = "0.6.7"
 authors = [
@@ -72,4 +65,4 @@ dynamic = ["dependencies", "readme"]
 
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}
-readme = {file = ["README.md"]}
+readme = {file = ["README.md"]}

requirements-dev.txt

@@ -4,6 +4,7 @@ coverage >= 6.3
 flask >= 2.0
 flit >= 3.8
 grayskull >= 1.1
+ipython >= 8.10.0 # not directly required, pinned by Snyk to avoid a vulnerability
 jupyterlab >= 3.3
 mistune >= 0.8
 mkdocs-git-revision-date-plugin >= 0.3
@@ -28,3 +29,6 @@ types-python-dateutil >= 2.8
 types-requests >= 2.27
 wheel >= 0.37
 xmltodict >= 0.12
+
+# test requirements for extra packages
+zarr >= 2.13.3

requirements.txt

@@ -27,5 +27,4 @@ scikit-learn >= 1.3.0
 scipy >= 1.8.0
 statsmodels >= 0.13
 tqdm >= 4.63
-urlpath >= 1.2
-
+urlpath >= 1.2

setup.py

@@ -55,6 +55,10 @@ def parse_requirements_file (
 
         return results
 
+with_zarr_extras = {
+   'with_zarr': ['simplejson>=3.5.3']
+}
+
 
 if __name__ == "__main__":
     spec = importlib.util.spec_from_file_location("kglab.version", "kglab/version.py")
@@ -76,14 +80,23 @@ def parse_requirements_file (
         long_description_content_type = "text/markdown",
 
         python_requires = ">=" + kglab_version._versify(kglab_version.MIN_PY_VERSION),  # pylint: disable=W0212
-        packages = setuptools.find_packages(exclude=[ "docs", "examples" ]),
         zip_safe = False,
 
+        packages = setuptools.find_packages(
+            exclude = [
+                "dat",
+                "docs",
+                "examples",
+                "scripts",
+                "tests",
+            ]),
+
         install_requires = base_packages,
         extras_require = {
             "base": base_packages,
             "docs": docs_packages,
             "tutorial": tut_packages,
+            "with-zarr": ["zarr>=2.13.3"]
         },
 
         author = "Paco Nathan",
@@ -114,12 +127,11 @@ def parse_requirements_file (
         url = "https://derwen.ai/docs/kgl/",
         project_urls = {
             "DOI": "https://doi.org/10.5281/zenodo.6360664",
-            "Community Survey": "https://forms.gle/FMHgtmxHYWocprMn6",
-            "Discussion Forum": "https://www.linkedin.com/groups/6725785/",
+            "Discussions": "https://www.linkedin.com/groups/6725785/",
             "DockerHub": "https://hub.docker.com/r/derwenai/kglab",
-            "Hands-on Tutorial": "https://derwen.ai/docs/kgl/tutorial/",
-            "Issue Tracker": "https://github.com/DerwenAI/kglab/issues",
-            "Source Code": "https://github.com/DerwenAI/kglab",
+            "Tutorial": "https://derwen.ai/docs/kgl/tutorial/",
+            "Issues": "https://github.com/DerwenAI/kglab/issues",
+            "Source": "https://github.com/DerwenAI/kglab",
         },
 
         entry_points = {

tests/test_algebra_basic.py

@@ -7,6 +7,7 @@
 from .__init__ import DAT_FILES_DIR
 
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 @pytest.fixture()
 def kg_test_data():
     namespaces = {
@@ -42,6 +43,7 @@ def get_items(s):
 }
 """
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 def test_adj_mtx(kg_test_data):
     subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
     n_array = subgraph.to_adjacency()
@@ -56,6 +58,7 @@ def test_adj_mtx(kg_test_data):
     )
 
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 def test_incidence(kg_test_data):
     subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
     n_array = subgraph.to_incidence()
@@ -69,6 +72,7 @@ def test_incidence(kg_test_data):
         rtol=1e-5, atol=0
     )
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 def test_laplacian(kg_test_data):
     subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
     n_array = subgraph.to_laplacian()
@@ -82,6 +86,7 @@ def test_laplacian(kg_test_data):
         rtol=1e-5, atol=0
     )
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 def test_scipy_sparse(kg_test_data):
     subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
     n_array = subgraph.to_scipy_sparse()
@@ -93,6 +98,7 @@ def test_scipy_sparse(kg_test_data):
     assert all(i in get_items(n_array) for i in set_)
     assert all(i not in get_items(n_array) for i in not_set_)
 
+@pytest.mark.skip(reason="rollback zarr dependency")
 def test_get_numbers(kg_test_data):
     subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
     subgraph.check_attributes()
@@ -101,3 +107,56 @@ def test_get_numbers(kg_test_data):
     assert subgraph._get_n_edges() == 1078
     assert subgraph.nx_graph.number_of_nodes() == 256
     assert subgraph.nx_graph.number_of_edges() == 1078
+
+
+@pytest.mark.skip(reason="rollback zarr dependency")
+def test_adj_mtx_with_zarr(kg_test_data):
+    subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
+    n_array = subgraph.to_zarr("to_adjacency")
+
+    assert(n_array.shape == (256, 256))
+
+    np.testing.assert_allclose(
+        n_array[:3,:6],
+        np.array(
+            [[0, 1, 1, 1, 1, 1],
+            [0, 0, 0, 0, 0, 0],
+            [0, 0, 0, 0, 0, 0]]
+        ),
+        rtol=1e-5, atol=0
+    )
+
+
+@pytest.mark.skip(reason="rollback zarr dependency")
+def test_incidence_with_zarr(kg_test_data):
+    subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
+    n_array = subgraph.to_zarr("to_incidence")
+
+    assert(n_array.shape == (256, 1078))
+
+    np.testing.assert_allclose(
+        n_array[:3,:6],
+        np.array(
+            [[1, 1, 1, 1, 1, 1],
+            [1, 0, 0, 0, 0, 0],
+            [0, 1, 0, 0, 0, 0]]
+        ),
+        rtol=1e-5, atol=0
+    )
+
+@pytest.mark.skip(reason="rollback zarr dependency")
+def test_laplacian_with_zarr(kg_test_data):
+    subgraph = SubgraphMatrix(kg=kg_test_data, sparql=QUERY1)
+    n_array = subgraph.to_zarr("to_laplacian")
+
+    assert(n_array.shape == (256, 256))
+
+    np.testing.assert_allclose(
+        n_array[:3,:6],
+        np.array(
+            [[6, -1, -1, -1, -1, -1],
+            [-1, 190, 0, 0, 0, 0],
+            [-1, 0, 147, 0, 0, 0]]
+        ),
+        rtol=1e-5, atol=0
+    )