Merge remote-tracking branch 'upstream/main' into samukweku/nth_function

.github/workflows/snyk-scan.yml

@@ -0,0 +1,66 @@
+name: Snyk Security Vulnerability Scan
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      - 'main'
+
+jobs:
+  snyk_scan_test:
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+
+      - uses: snyk/actions/setup@master
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Check changed Deps files
+        uses: tj-actions/changed-files@v35
+        id: changed-files
+        with:
+          files: |
+            **/requirements*.txt
+
+      - name: Snyk scan for Python dependencies
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            python3 -m pip install -r $file
+            snyk test --command=python3 --package-manager=pip --file=$file -d --skip-unresolved 
+          done
+  
+  snyk_scan_monitor:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Extract github branch/tag name
+        shell: bash
+        run: echo "ref=$(echo ${GITHUB_REF##*/})" >> $GITHUB_OUTPUT
+        id: extract_ref
+
+      - uses: actions/checkout@master
+
+      - uses: snyk/actions/setup@master
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Snyk scan for Python dependencies
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        run: |
+          for file in requirements*.txt; do
+            python3 -m pip install -r $file
+            snyk monitor --command=python3 --skip-unresolved --org=py_data_table --remote-repo-url=datatable/${{ steps.extract_ref.outputs.ref }} --package-manager=pip --file=$file --project-name=PY-DATA-TABLE/datatable/${{ steps.extract_ref.outputs.ref }}/$file -d --fail-on=all
+          done

ci/Jenkinsfile.groovy

@@ -213,7 +213,7 @@ ansiColor('xterm') {
                                 -e DT_BUILD_NUMBER=${DT_BUILD_NUMBER} \
                                 --entrypoint /bin/bash \
                                 ${DOCKER_IMAGE_X86_64_MANYLINUX} \
-                                -c "env && /opt/python/cp37-cp37m/bin/python3.7 ci/ext.py sdist"
+                                -c "env && /opt/python/cp38-cp38/bin/python3.8 ci/ext.py sdist"
                         """
                         sh """
                             echo "--------- _build_info.py --------------------"
@@ -251,15 +251,13 @@ ansiColor('xterm') {
                                         -c "cd /dot && \
                                             ls -la && \
                                             ls -la src/datatable && \
-                                            /opt/python/cp37-cp37m/bin/python3.7 ci/ext.py wheel --audit && \
                                             /opt/python/cp38-cp38/bin/python3.8 ci/ext.py debugwheel --audit && \
                                             /opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
                                             /opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
                                             /opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
                                             /opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
                                             echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
                                             mv dist/*debug*.whl . && \
-                                            echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
                                             echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
                                             echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
                                             echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
@@ -286,9 +284,6 @@ ansiColor('xterm') {
                                          "DT_BUILD_SUFFIX=${DT_BUILD_SUFFIX}",
                                          "DT_BUILD_NUMBER=${DT_BUILD_NUMBER}"]) {
                                     sh """
-                                        source /Users/jenkins/datatable_envs/py37/bin/activate
-                                        python ci/ext.py wheel
-                                        deactivate
                                         source /Users/jenkins/datatable_envs/py38/bin/activate
                                         python ci/ext.py wheel
                                         deactivate
@@ -301,7 +296,6 @@ ansiColor('xterm') {
                                         source /Users/jenkins/datatable_envs/py311/bin/activate
                                         python ci/ext.py wheel
                                         deactivate
-                                        echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py
                                         echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py
                                         echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py
                                         echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py
@@ -341,15 +335,13 @@ ansiColor('xterm') {
                                             -c "cd /dot && \
                                                 ls -la && \
                                                 ls -la src/datatable && \
-                                                /opt/python/cp37-cp37m/bin/python3.7 ci/ext.py wheel --audit && \
                                                 /opt/python/cp38-cp38/bin/python3.8 ci/ext.py debugwheel --audit && \
                                                 /opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
                                                 /opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
                                                 /opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
                                                 /opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
                                                 echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
                                                 mv dist/*debug*.whl . && \
-                                                echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
@@ -373,20 +365,6 @@ ansiColor('xterm') {
             if (!params.DISABLE_ALL_TESTS) {
                 def testStages = [:]
                 testStages <<
-                    namedStage('Test x86_64-manylinux-py37', { stageName, stageDir ->
-                        node(NODE_LINUX) {
-                            buildSummary.stageWithSummary(stageName, stageDir) {
-                                cleanWs()
-                                dumpInfo()
-                                dir(stageDir) {
-                                    unstash 'datatable-sources'
-                                    unstash 'x86_64-manylinux-wheels'
-                                    test_in_docker("x86_64-manylinux-py37", "37",
-                                                   DOCKER_IMAGE_X86_64_MANYLINUX)
-                                }
-                            }
-                        }
-                    }) <<
                     namedStage('Test x86_64-manylinux-py38-debug', doPy38Tests, { stageName, stageDir ->
                         node(NODE_LINUX) {
                             buildSummary.stageWithSummary(stageName, stageDir) {
@@ -457,20 +435,6 @@ ansiColor('xterm') {
                             }
                         }
                     }) <<
-                    namedStage('Test ppc64le-manylinux-py37', doPpcTests, { stageName, stageDir ->
-                        node(NODE_PPC) {
-                            buildSummary.stageWithSummary(stageName, stageDir) {
-                                cleanWs()
-                                dumpInfo()
-                                dir(stageDir) {
-                                    unstash 'datatable-sources'
-                                    unstash 'ppc64le-manylinux-wheels'
-                                    test_in_docker("ppc64le-manylinux-py37", "37",
-                                                   DOCKER_IMAGE_PPC64LE_MANYLINUX)
-                                }
-                            }
-                        }
-                    }) <<
                     namedStage('Test ppc64le-manylinux-py38-debug', doPpcTests && doPy38Tests, { stageName, stageDir ->
                         node(NODE_PPC) {
                             buildSummary.stageWithSummary(stageName, stageDir) {
@@ -541,19 +505,6 @@ ansiColor('xterm') {
                             }
                         }
                     }) <<
-                    namedStage('Test x86_64-macos-py37', { stageName, stageDir ->
-                        node(NODE_MACOS) {
-                            buildSummary.stageWithSummary(stageName, stageDir) {
-                                cleanWs()
-                                dumpInfo()
-                                dir(stageDir) {
-                                    unstash 'datatable-sources'
-                                    unstash 'x86_64-macos-wheels'
-                                    test_macos('37')
-                                }
-                            }
-                        }
-                    }) <<
                     namedStage('Test x86_64-macos-py38', doPy38Tests, { stageName, stageDir ->
                         node(NODE_MACOS) {
                             buildSummary.stageWithSummary(stageName, stageDir) {
@@ -756,7 +707,7 @@ ansiColor('xterm') {
 //     used as a prefix for the test-report file name.
 //
 // pyver
-//     python version string, such as "37" or "310"
+//     python version string, such as "38" or "310"
 //
 // docker_image
 //     Name of the docker container where the tests will be run
@@ -824,7 +775,6 @@ def test_in_docker(String testtag, String pyver, String docker_image) {
 
 def get_python_for_docker(String pyver, String image) {
     if (image == DOCKER_IMAGE_X86_64_MANYLINUX || image == DOCKER_IMAGE_PPC64LE_MANYLINUX) {
-        if (pyver == "37") return "/opt/python/cp37-cp37m/bin/python3.7"
         if (pyver == "38") return "/opt/python/cp38-cp38/bin/python3.8"
         if (pyver == "39") return "/opt/python/cp39-cp39/bin/python3.9"
         if (pyver == "310") return "/opt/python/cp310-cp310/bin/python3.10"

ci/appveyor.yml

@@ -183,40 +183,6 @@ build_script:
 
 
 
-    # =======================================================================
-    #   Build and test wheel for Python 3.7
-    # =======================================================================
-
-    $env:PATH = "C:/Python37-x64;C:/Python37-x64/Scripts;$DEFAULT_PATH"
-
-    python -V
-
-    python ci/ext.py wheel
-
-    $DT_WHEEL = ls dist/*-cp37-*.whl
-
-    echo "DT_WHEEL = $DT_WHEEL"
-
-    echo "----- _build_info.py for Python 3.7 ------------------------------"
-
-    cat src/datatable/_build_info.py
-
-    echo "------------------------------------------------------------------"
-
-    python -m pip install --upgrade pip
-
-    python -m pip install $DT_WHEEL
-
-    python -m pip install pytest docutils pandas pyarrow
-
-    python -m pytest -ra --maxfail=10 -Werror -vv --showlocals ./tests/
-
-    if(!$?) { Exit $LASTEXITCODE }
-
-    python -m pip uninstall -y $DT_WHEEL
-
-
-
     # =======================================================================
     #   Build and test wheel for Python 3.8
     # =======================================================================

ci/ext.py

@@ -412,9 +412,10 @@ def get_meta():
             "Operating System :: Microsoft :: Windows",
             "Operating System :: Unix",
             "Programming Language :: Python :: 3 :: Only",
-            "Programming Language :: Python :: 3.6",
-            "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
+            "Programming Language :: Python :: 3.10",
+            "Programming Language :: Python :: 3.11",
             "Topic :: Scientific/Engineering :: Information Analysis",
         ],
 

ci/xbuild/extension.py

@@ -344,7 +344,7 @@ def xbuild_version(self):
     def pyabi(self):
         """
         Python `SOABI` config variable, which may look something like
-        'cpython-37m-darwin'. If this does not correspond to the value
+        'cpython-38m-darwin'. If this does not correspond to the value
         stored in .xbuild, then all sources will be rebuilt.
         """
         return sysconfig.get_config_var("SOABI")

docs/releases/v1.1.0.rst

@@ -62,6 +62,9 @@
 
     -[fix] Fixed creating frames from numpy arrays, that contain unicode strings. [#3420]
 
+    -[fix] :meth:`.to_numpy()` will now create a correctly shaped array
+        in the case of zero-column frames. [#3427]
+
     -[api] Converting a column of :attr:`void <dt.Type.void>` type into pandas
         now produces a pandas ``object`` column filled with ``None``s. Converting
         such column back into datatable produces a ``void`` column again. [#3063]
@@ -191,11 +194,16 @@
     General
     -------
 
-    -[api] Datatable no longer supports Python 3.6, because Python 3.6 itself
-      has reached its end of life on 2021-12-23 and will no longer be
-      supported. If you are still using Python 3.6, please consider upgrading.
+    -[api] Datatable no longer supports Python 3.6, because it
+      has reached its end of life on 2021-12-23 and will no longer be supported.
+      If you are still using Python 3.6, please consider upgrading.
       [#3376]
 
+    -[api] Datatable no longer supports Python 3.7, because it
+      has reached its end of life on 2023-06-27 and will no longer be supported.
+      If you are still using Python 3.7, please consider upgrading.
+      [#3434]
+
     -[new] Added properties :attr:`.is_array <dt.Type.is_array>`,
         :attr:`.is_boolean <dt.Type.is_boolean>`,
         :attr:`.is_categorical <dt.Type.is_categorical>`,

docs/start/install.rst

@@ -9,13 +9,13 @@ This page describes how to install ``datatable`` on various systems.
 Prerequisites
 -------------
 
-Python 3.7+ is required. Generally, we will support each version of Python
+Python 3.8+ is required. Generally, we will support each version of Python
 until its official `end of life`_. You can verify your python version via
 
 .. code-block:: console
 
     $ python --version
-    Python 3.7.10
+    Python 3.8.10
 
 In addition, we recommend using ``pip`` version 20.3+, especially if you're
 planning to install datatable from the source, or if you are on a Unix machine.
@@ -233,7 +233,7 @@ know how to resolve them. If none of these help you, please ask a question on
 ``Python.h: no such file or directory`` when compiling from source
   Your Python distribution was shipped without the ``Python.h`` header file.
   This has been observed on certain Linux machines. You would need to install
-  a Python package with a ``-dev`` suffix, for example ``python3.7-dev``.
+  a Python package with a ``-dev`` suffix, for example ``python3.8-dev``.
 
 ``fatal error: 'sys/mman.h' file not found`` on macOS
   In order to compile from source on mac computers, you need to have Xcode

src/core/frame/to_numpy.cc

@@ -1,5 +1,5 @@
 //------------------------------------------------------------------------------
-// Copyright 2018-2022 H2O.ai
+// Copyright 2018-2023 H2O.ai
 //
 // Permission is hereby granted, free of charge, to any person obtaining a
 // copy of this software and associated documentation files (the "Software"),
@@ -71,8 +71,8 @@ oobj Frame::to_numpy(const PKArgs& args) {
       col.cast_inplace(target_type);
     }
     auto res = to_numpy_impl(
-        Frame::oframe(new DataTable({col}, DataTable::default_names)),
-        c_contiguous
+      Frame::oframe(new DataTable({col}, DataTable::default_names)),
+      c_contiguous
     );
     return res.invoke("reshape", {oint(col.nrows())});
   }
@@ -103,7 +103,7 @@ static oobj to_numpy_impl(oobj frame, bool c_contiguous) {
   size_t ncols = dt->ncols();
   if (ncols == 0) {
     otuple shape(2);
-    shape.set(0, oint(0));
+    shape.set(0, frame.get_attr("nrows").to_pyint());
     shape.set(1, oint(0));
     return numpy.invoke("empty", {shape});
   }

src/core/parallel/api_primitives.h

@@ -1,5 +1,5 @@
 //------------------------------------------------------------------------------
-// Copyright 2019 H2O.ai
+// Copyright 2019-2023 H2O.ai
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,14 +19,20 @@
 #include "utils/assert.h"
 namespace dt {
 
-size_t num_threads_in_pool();
+#ifndef DT_DEFAULT_CHUNK_SIZE
+  #define DT_DEFAULT_CHUNK_SIZE 1000
+#endif
 
+#ifndef DT_DEFAULT_MIN_ITERS_PER_THREAD
+  #define DT_DEFAULT_MIN_ITERS_PER_THREAD 1000
+#endif
 
+size_t num_threads_in_pool();
 
 class ChunkSize {
   size_t value;
   public:
-    explicit ChunkSize(size_t sz = 1000 /* Default chunk size */) {
+    explicit ChunkSize(size_t sz = DT_DEFAULT_CHUNK_SIZE) {
       value = sz? sz : 1;
     }
     ChunkSize(const ChunkSize&) = default;
@@ -68,7 +74,7 @@ class NThreads {
  * iterations per thread.
  */
 NThreads nthreads_from_niters(size_t niters,
-                              size_t min_iters_per_thread = 1000,
+                              size_t min_iters_per_thread = DT_DEFAULT_MIN_ITERS_PER_THREAD,
                               bool parallel_ok = true);
 
 

src/core/parallel/job_idle.cc

@@ -20,9 +20,6 @@
 // IN THE SOFTWARE.
 //------------------------------------------------------------------------------
 #include "utils/assert.h"
-#ifndef DT_DISABLE
-  #include "utils/exceptions.h"
-#endif
 #include "parallel/api.h"
 #include "parallel/job_idle.h"
 #include "parallel/thread_pool.h"