notebook/0.1.4/rust_acceleration_file_line_count: updater

notebook/0.1.4/rust_acceleration_file_line_count.ipynb

@@ -9,14 +9,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
     "import os\n",
     "from pathlib import Path\n",
-    "from pyeio import opt\n",
-    "from pyeio.opt import comparison"
+    "from pyeio import opt"
    ]
   },
   {
@@ -28,7 +27,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -40,17 +39,95 @@
     }
    ],
    "source": [
-    "file = Path.home() / \"user/project/generic/personal/entitect/local/sample/items.jsonl\"\n",
-    "assert file.exists()\n",
-    "print(file.name, \"= {:,} bytes\".format(os.path.getsize(file)))"
+    "fp = Path.home() / \"user/project/generic/personal/entitect/local/sample/items.jsonl\"\n",
+    "assert fp.exists()\n",
+    "print(fp.name, \"= {:,} bytes\".format(os.path.getsize(fp)))\n",
+    "fp = str(fp)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Optimized but improved function is exposed here:\n",
+    "\n",
+    "```python\n",
+    "opt.count_lines_in_file(...)\n",
+    "```\n",
+    "\n",
+    "But can also directly access rust functions with:\n",
+    "\n",
+    "```python\n",
+    "opt.rs.count_lines_in_file(...)\n",
+    "```\n",
+    "\n",
+    "Or python comparison/fallback functions with:\n",
+    "\n",
+    "```python\n",
+    "opt.py.count_lines_in_file(...)\n",
+    "```"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "6.02 s ± 42.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "opt.py.count_lines_in_file(path=fp, chunk_size=1<<20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "382 ms ± 8.35 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%timeit\n",
+    "opt.rs.count_lines_in_file(path=fp, chunk_size=1<<20, num_threads=8)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "rust is ~16x faster\n"
+     ]
+    }
+   ],
+   "source": [
+    "speed_increase = round(6.02 / 0.382)\n",
+    "print(f\"rust is ~{speed_increase}x faster\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Can probably be optimized more."
+   ]
   }
  ],
  "metadata": {