mc4s/clean_dev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "\"\"\"\n",
    "mc4を掃除して､fasttextでgood/badを分類して､クラスタリングして記録するサンプルコード\n",
    "annotationもこのnotebookで行います\n",
    "\n",
    "\"\"\"\n",
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "#mc4の読み込み\n",
    "mc4_dataset = load_dataset('mc4', 'ja',split='train', streaming=True)\n",
    "\n",
    "\"\"\"\n",
    "#oscarなども読み込める\n",
    "#ignore_verifications=Trueをつけないとエラーとなる\n",
    "oscar_dataset = load_dataset('oscar', 'unshuffled_original_ja', \n",
    "                       split='train', \n",
    "                       ignore_verifications=True,\n",
    "                       streaming=True)\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "dataset=mc4_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.cleaner.auto_cleaner import clean_text\n",
    "from src.classifier.DatasetAnnotator import DatasetAnnotator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotator=DatasetAnnotator(dataset,clean_func=clean_text,n_preload=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#annotations\n",
    "annotator.ask_annotations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fasttextで訓練\n",
    "annotator.train_fasttext(autotuneDuration=120,wordNgrams=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#テキストの教師なしクラスタリング: 結局使わない\n",
    "\n",
    "#modelをannotations/text_labelsに保存しておく\n",
    "#!wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ja.300.bin.gz\n",
    "#!gzip -d cc.ja.300.bin.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.classifier.NounClustering import NounClustering\n",
    "#clf=NounClustering()\n",
    "\n",
    "#clf.train_wiki(n_samples=10**8) #wikipediaのタイトルで教師なしクラスタリングの学習"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#モデルの挙動確認\n",
    "#カテゴリの分類がかなりいまいち｡\n",
    "\"\"\"\n",
    "for i in range(100):\n",
    "    text=annotator.get_text_by_id(i)\n",
    "    text_=text.replace(\"\\n\",\"\")\n",
    "    if text==\"\":continue\n",
    "    is_noise=(annotator.predict(text))\n",
    "    genre=clf.predict(text)\n",
    "    print(is_noise,genre,text_[:300])\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_split=10**2\n",
    "import os\n",
    "\n",
    "corpus_dir=\"corpus/test\"\n",
    "if not os.path.exists(corpus_dir):\n",
    "    os.makedirs(corpus_dir)\n",
    "\n",
    "cnt=0\n",
    "record_id=-1\n",
    "for record in dataset:\n",
    "    record_id+=1\n",
    "    text=record['text']\n",
    "\n",
    "    #テキストクリーン\n",
    "    text=clean_text(text)\n",
    "    if text==\"\":\n",
    "        continue\n",
    "\n",
    "    #記事の判定\n",
    "    is_noise=annotator.predict(text)\n",
    "    if is_noise==1:\n",
    "        continue\n",
    "\n",
    "    #ジャンルの判定\n",
    "    #genre=clf.predict(text)\n",
    "    cnt+=1\n",
    "\n",
    "    d={\n",
    "        \"id\":record_id,\n",
    "        #\"cat\":genre,\n",
    "        \"text\":text,\n",
    "    }\n",
    "    file_name=f\"{corpus_dir}/{cnt//n_split}.txt\"\n",
    "    with open(file_name, \"a\") as f:\n",
    "        f.write(str(d)+\"\\n\")\n",
    "\n",
    "    if cnt>1000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#圧縮率\n",
    "cnt/record_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "crawl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}