From 3839e6e8c39041826c4a36403a1fdb75662c7026 Mon Sep 17 00:00:00 2001 From: Roger-Luo Date: Wed, 11 Aug 2021 17:02:57 -0400 Subject: [PATCH 01/10] support sparse array conversion --- Project.toml | 1 + src/conversions.jl | 3 +++ src/scipy.jl | 19 +++++++++++++++++++ test/runtests.jl | 6 ++++++ 4 files changed, 29 insertions(+) create mode 100644 src/scipy.jl diff --git a/Project.toml b/Project.toml index 0561b43d..b71da446 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" [compat] diff --git a/src/conversions.jl b/src/conversions.jl index 0226f036..c183b98c 100644 --- a/src/conversions.jl +++ b/src/conversions.jl @@ -426,6 +426,9 @@ PyObject(a::BitArray) = PyObject(Array(a)) # NumPy conversions (multidimensional arrays) include("numpy.jl") +# SciPy conversions +include("scipy.jl") + ######################################################################### # PyDict: no-copy wrapping of a Julia object around a Python dictionary diff --git a/src/scipy.jl b/src/scipy.jl new file mode 100644 index 00000000..c38df0b8 --- /dev/null +++ b/src/scipy.jl @@ -0,0 +1,19 @@ +const scipysparse_ = PyNULL() + +function scipysparse() + if ispynull(scipysparse_) + copy!(scipysparse_, pyimport_conda("scipy.sparse", "scipy")) + end + return scipysparse_ +end + +using SparseArrays + +function PyObject(S::SparseMatrixCSC) + scipysparse()["csc_matrix"]((S.nzval, S.rowval .- 1, S.colptr .- 1), shape=size(S)) +end + +function convert(::Type{SparseMatrixCSC}, o::PyObject) + I, J, V = scipysparse().find(o) + return sparse(I .+ 1, J .+ 1, V) +end diff --git a/test/runtests.jl b/test/runtests.jl index 2e826080..b019a8ce 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -615,6 +615,12 @@ const PyInt = pyversion < v"3" ? Int : Clonglong @test float(PyObject(1+2im)) === 1.0 + 2.0im @test float(PyObject([1,2,3]))[2] === 2.0 @test_throws ArgumentError float(pybuiltin("type")) + + # sparse array conversion + import SparseArrays: SparseMatrixCSC, sparse, sprand + S = sprand(10, 10, 0.2) + PyS = PyObject(S) + @test convert(SparseMatrixCSC, PyS) == S end ###################################################################### From 06a6c33c5dcd2e361d4aee37e8c28c5e37d91533 Mon Sep 17 00:00:00 2001 From: Roger-Luo Date: Wed, 11 Aug 2021 17:19:43 -0400 Subject: [PATCH 02/10] fix shape --- src/scipy.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scipy.jl b/src/scipy.jl index c38df0b8..3f79fffb 100644 --- a/src/scipy.jl +++ b/src/scipy.jl @@ -15,5 +15,5 @@ end function convert(::Type{SparseMatrixCSC}, o::PyObject) I, J, V = scipysparse().find(o) - return sparse(I .+ 1, J .+ 1, V) + return sparse(I .+ 1, J .+ 1, V, o.shape...) end From b1a461cc644c6b709b2f4c5e0fd70f20945efbc5 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 22:58:32 -0400 Subject: [PATCH 03/10] Install SciPy on CI --- .github/workflows/aot.yml | 2 +- .github/workflows/system.yml | 2 +- test/runtests.jl | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index ea23d765..b4cb9062 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -33,7 +33,7 @@ jobs: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.architecture }} - run: python --version - - run: python -m pip install --user numpy + - run: python -m pip install --user numpy scipy - name: Setup julia uses: julia-actions/setup-julia@v1 with: diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index e52ef8de..f24bbd71 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -49,7 +49,7 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.architecture }} - - run: python -m pip install --user numpy + - run: python -m pip install --user numpy scipy - run: python -m pip install virtualenv # virtualenv test with Python 2.7 is failing for some reason. # Skipping it for now. diff --git a/test/runtests.jl b/test/runtests.jl index b019a8ce..d5221843 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using PyCall using PyCall: hasproperty using Test, Dates, Serialization +import Conda filter(f, itr) = collect(Iterators.filter(f, itr)) filter(f, d::AbstractDict) = Base.filter(f, d) @@ -13,6 +14,10 @@ PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","") @testset "CI setup" begin if lowercase(get(ENV, "CI", "false")) == "true" @test !ispynull(pyimport_e("numpy")) + if PyCall.conda && ispynull(pyimport_e("scipy.sparse")) + Conda.add("scipy") + end + @test !ispynull(pyimport_e("scipy.sparse")) end end From d115bbb037285651c265550d6b8b298c3a8bb82c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:15:03 -0400 Subject: [PATCH 04/10] Simplify SciPy installation using Conda --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d5221843..1af62cc6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,6 @@ using PyCall using PyCall: hasproperty using Test, Dates, Serialization -import Conda filter(f, itr) = collect(Iterators.filter(f, itr)) filter(f, d::AbstractDict) = Base.filter(f, d) @@ -14,8 +13,9 @@ PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","") @testset "CI setup" begin if lowercase(get(ENV, "CI", "false")) == "true" @test !ispynull(pyimport_e("numpy")) - if PyCall.conda && ispynull(pyimport_e("scipy.sparse")) - Conda.add("scipy") + try + pyimport_conda("scipy.sparse", "scipy") + catch end @test !ispynull(pyimport_e("scipy.sparse")) end From 874f5df2d5e661016b1ccac83dc1a3f5cff02467 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:15:48 -0400 Subject: [PATCH 05/10] Don't use pyimport_conda in lazy load This seems to be compatible with other uses in PyCall.jl. Also, doing accessing internet when invoking `convert` is rather wacky. --- src/scipy.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scipy.jl b/src/scipy.jl index 3f79fffb..1e82676d 100644 --- a/src/scipy.jl +++ b/src/scipy.jl @@ -2,7 +2,7 @@ const scipysparse_ = PyNULL() function scipysparse() if ispynull(scipysparse_) - copy!(scipysparse_, pyimport_conda("scipy.sparse", "scipy")) + copy!(scipysparse_, pyimport("scipy.sparse")) end return scipysparse_ end From 4f8f592266299eee0cae7290052c40413310fe81 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:18:24 -0400 Subject: [PATCH 06/10] Import at the opt of the file --- src/scipy.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scipy.jl b/src/scipy.jl index 1e82676d..744841fe 100644 --- a/src/scipy.jl +++ b/src/scipy.jl @@ -1,3 +1,5 @@ +using SparseArrays + const scipysparse_ = PyNULL() function scipysparse() @@ -7,8 +9,6 @@ function scipysparse() return scipysparse_ end -using SparseArrays - function PyObject(S::SparseMatrixCSC) scipysparse()["csc_matrix"]((S.nzval, S.rowval .- 1, S.colptr .- 1), shape=size(S)) end From a87ced5b4479435d245ec816a1d7e7efb0bed623 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:28:57 -0400 Subject: [PATCH 07/10] Use getproperty always --- src/scipy.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scipy.jl b/src/scipy.jl index 744841fe..df581fc2 100644 --- a/src/scipy.jl +++ b/src/scipy.jl @@ -10,7 +10,7 @@ function scipysparse() end function PyObject(S::SparseMatrixCSC) - scipysparse()["csc_matrix"]((S.nzval, S.rowval .- 1, S.colptr .- 1), shape=size(S)) + scipysparse().csc_matrix((S.nzval, S.rowval .- 1, S.colptr .- 1), shape = size(S)) end function convert(::Type{SparseMatrixCSC}, o::PyObject) From 6629f2919e17d85c87e6fffb16b0a2eb16f3c570 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:41:56 -0400 Subject: [PATCH 08/10] Don't use `import M: f` --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1af62cc6..0c1a1e08 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using PyCall using PyCall: hasproperty using Test, Dates, Serialization +using SparseArrays: SparseMatrixCSC, sparse, sprand filter(f, itr) = collect(Iterators.filter(f, itr)) filter(f, d::AbstractDict) = Base.filter(f, d) @@ -622,7 +623,6 @@ const PyInt = pyversion < v"3" ? Int : Clonglong @test_throws ArgumentError float(pybuiltin("type")) # sparse array conversion - import SparseArrays: SparseMatrixCSC, sparse, sprand S = sprand(10, 10, 0.2) PyS = PyObject(S) @test convert(SparseMatrixCSC, PyS) == S From fecc4c270960c4c6608652b4ddc45b3608847560 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 23 Oct 2021 23:45:18 -0400 Subject: [PATCH 09/10] DEBUG: Rollback to Python 3.9 --- .github/workflows/system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index f76089f0..be6d4766 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -18,7 +18,7 @@ jobs: - macos-latest - windows-latest architecture: [x64] - python-version: ['3.x'] + python-version: ['3.9'] julia-version: - '1.0' - '1' From f04535a6028aa15afb2cfe6ff1dc825502676432 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sun, 24 Oct 2021 00:13:58 -0400 Subject: [PATCH 10/10] Revert "DEBUG: Rollback to Python 3.9" This reverts commit fecc4c270960c4c6608652b4ddc45b3608847560. --- .github/workflows/system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index be6d4766..f76089f0 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -18,7 +18,7 @@ jobs: - macos-latest - windows-latest architecture: [x64] - python-version: ['3.9'] + python-version: ['3.x'] julia-version: - '1.0' - '1'