Testing_in-features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2eb74832-12aa-4e35-9c84-7e3d9d415579",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import os.path\n",
    "import pathlib\n",
    "import glob\n",
    "from pathlib import Path\n",
    "import glob\n",
    "import psutil\n",
    "import math\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "import tensorflow_io as tfio\n",
    "\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "from IPython import display\n",
    "\n",
    "# Set the seed value for experiment reproducibility.\n",
    "seed = 4385789\n",
    "tf.random.set_seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "gpus = tf.config.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "  try:\n",
    "    # Currently, memory growth needs to be the same across GPUs\n",
    "    for gpu in gpus:\n",
    "      tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    logical_gpus = tf.config.list_logical_devices('GPU')\n",
    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "  except RuntimeError as e:\n",
    "    # Memory growth must be set before GPUs have been initialized\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c1a043b7-f16c-4cfe-b111-ca5bce58b4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'Z:/AutoMusic/output_3/'\n",
    "cache_dir = 'Z:/AutoMusic/cache_3/'\n",
    "test_file = data_dir+\"/high_up/high_up.02-block_and_crown-remember_the_good_times_(_057672_064481.wav\"\n",
    "pioneer_files = 'd:/pioneer/usbanlz'\n",
    "pioneer_prefix = 'D:'\n",
    "temp_wav_file = 'Z:/AutoMusic/temp/temp.wav'\n",
    "mytotalfiles = 934 # I know this from previous runs.\n",
    "EPOCHS = 200 # using early stopping, but just in case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3c7c76e0-ca62-452d-8e33-87c69329288a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using run history file: ./Histories/history39.txt\n",
      "Using model save/checkpoint file: Z:/AutoMusic/checkpoint/automusic39.h5\n",
      "Sample size of 65GB should fit in free memory of 94GB - using RAM to cache\n"
     ]
    }
   ],
   "source": [
    "# Find the next history file number\n",
    "\n",
    "historycounter = 1 # start looking at this number as history1.txt\n",
    "\n",
    "Path(\"./Histories\").mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "while os.path.isfile(\"./Histories/history\"+str(historycounter)+\".txt\"):\n",
    "    historycounter += 1\n",
    "\n",
    "history_file = \"./Histories/history\"+str(historycounter)+\".txt\"\n",
    "print(\"Using run history file: \"+history_file)\n",
    "\n",
    "checkpoint_filepath = 'Z:/AutoMusic/checkpoint/automusic'+str(historycounter)+'.h5'\n",
    "print(\"Using model save/checkpoint file: \"+checkpoint_filepath)\n",
    "\n",
    "# Roughly figure out if the dataset will fit into memory, or we need to use disk caching\n",
    "\n",
    "free_mem = psutil.virtual_memory()\n",
    "free_mem = math.floor(free_mem.available/1024/1024/1024*0.90)\n",
    "\n",
    "samples_size = sum(f.stat().st_size for f in Path(data_dir).glob('**/*') if f.is_file())\n",
    "samples_size = math.ceil((samples_size/1024/1024/1024)*1.05)\n",
    "\n",
    "if samples_size < free_mem:\n",
    "    print(\"Sample size of \"+str(samples_size)+\"GB should fit in free memory of \"+str(free_mem)+\"GB - using RAM to cache\")\n",
    "    cache_dir = ''\n",
    "else:\n",
    "    print(\"Sample size of \"+str(samples_size)+\"GB will not fit in free memory of \"+str(int(free_mem))+\"GB - using cache dir \"+cache_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "00469c98-ca2c-477e-85ba-474d94216d8a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directories/Labels: ['high_chorus' 'high_down' 'high_intro' 'high_outro' 'high_up']\n"
     ]
    }
   ],
   "source": [
    "commands = np.array(tf.io.gfile.listdir(str(data_dir)))\n",
    "commands = commands[(commands != 'README.md') & (commands != '.DS_Store')]\n",
    "print('Directories/Labels:', commands)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0224dfcb-b4c8-4518-8d02-c3e8aa458ed6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 123264 files belonging to 5 classes.\n",
      "Using 98612 files for training.\n",
      "Using 24652 files for validation.\n",
      "\n",
      "label names: ['high_chorus' 'high_down' 'high_intro' 'high_outro' 'high_up']\n"
     ]
    }
   ],
   "source": [
    "train_ds, val_ds = tf.keras.utils.audio_dataset_from_directory(\n",
    "    directory=data_dir,\n",
    "    batch_size=32,\n",
    "    validation_split=0.2,\n",
    "    seed=0,\n",
    "    output_sequence_length=44100*3,\n",
    "    subset='both')\n",
    "\n",
    "label_names = np.array(train_ds.class_names)\n",
    "print()\n",
    "print(\"label names:\", label_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6cc5bfb-2936-4c1d-a7b7-fa74740943c2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def squeeze(audio, labels):\n",
    "  # audio = tf.squeeze(audio, axis=-1)\n",
    "  audio = audio[:,:,-1]\n",
    "  return audio, labels\n",
    "\n",
    "train_ds = train_ds.map(squeeze, num_parallel_calls=tf.data.AUTOTUNE)\n",
    "val_ds = val_ds.map(squeeze, num_parallel_calls=tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ded6addc-4093-42a8-952a-288d2efbb695",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_ds = val_ds.shard(num_shards=2, index=0)\n",
    "val_ds = val_ds.shard(num_shards=2, index=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f69c9591-66b5-431a-b5cf-132ec605fe25",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 132300)\n",
      "(32,)\n"
     ]
    }
   ],
   "source": [
    "for example_audio, example_labels in train_ds.take(1):  \n",
    "  print(example_audio.shape)\n",
    "  print(example_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9c0ff93e-17b3-4a6b-9129-63463d9e3589",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class LogMelSpectrogram(tf.keras.layers.Layer):\n",
    "    \"\"\"Compute log-magnitude mel-scaled spectrograms.\"\"\"\n",
    "\n",
    "    def __init__(self, sample_rate, fft_size, hop_size, n_mels,\n",
    "                 f_min=0.0, f_max=None, **kwargs):\n",
    "        super(LogMelSpectrogram, self).__init__(**kwargs)\n",
    "        self.sample_rate = sample_rate\n",
    "        self.fft_size = fft_size\n",
    "        self.hop_size = hop_size\n",
    "        self.n_mels = n_mels\n",
    "        self.f_min = f_min\n",
    "        self.f_max = f_max if f_max else sample_rate / 2\n",
    "        self.mel_filterbank = tf.signal.linear_to_mel_weight_matrix(\n",
    "            num_mel_bins=self.n_mels,\n",
    "            num_spectrogram_bins=fft_size // 2 + 1,\n",
    "            sample_rate=self.sample_rate,\n",
    "            lower_edge_hertz=self.f_min,\n",
    "            upper_edge_hertz=self.f_max)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.non_trainable_weights.append(self.mel_filterbank)\n",
    "        super(LogMelSpectrogram, self).build(input_shape)\n",
    "\n",
    "    def call(self, waveforms):\n",
    "        \"\"\"Forward pass.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        waveforms : tf.Tensor, shape = (None, n_samples)\n",
    "            A Batch of mono waveforms.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        log_mel_spectrograms : (tf.Tensor), shape = (None, time, freq, ch)\n",
    "            The corresponding batch of log-mel-spectrograms\n",
    "        \"\"\"\n",
    "        def _tf_log10(x):\n",
    "            numerator = tf.math.log(x)\n",
    "            denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))\n",
    "            return numerator / denominator\n",
    "\n",
    "        def power_to_db(magnitude, amin=1e-16, top_db=80.0):\n",
    "            \"\"\"\n",
    "            https://librosa.github.io/librosa/generated/librosa.core.power_to_db.html\n",
    "            I think this is a function in the TF supports now?\n",
    "            \"\"\"\n",
    "            ref_value = tf.reduce_max(magnitude)\n",
    "            log_spec = 10.0 * _tf_log10(tf.maximum(amin, magnitude))\n",
    "            log_spec -= 10.0 * _tf_log10(tf.maximum(amin, ref_value))\n",
    "            log_spec = tf.maximum(log_spec, tf.reduce_max(log_spec) - top_db)\n",
    "\n",
    "            return log_spec\n",
    "\n",
    "        spectrograms = tf.signal.stft(waveforms,\n",
    "                                      frame_length=self.fft_size,\n",
    "                                      frame_step=self.hop_size,\n",
    "                                      pad_end=False)\n",
    "\n",
    "        magnitude_spectrograms = tf.abs(spectrograms)\n",
    "\n",
    "        mel_spectrograms = tf.matmul(tf.square(magnitude_spectrograms),\n",
    "                                     self.mel_filterbank)\n",
    "\n",
    "        log_mel_spectrograms = power_to_db(mel_spectrograms)\n",
    "\n",
    "        # add channel dimension\n",
    "        log_mel_spectrograms = tf.expand_dims(log_mel_spectrograms, 3)\n",
    "\n",
    "        return log_mel_spectrograms\n",
    "\n",
    "    def get_config(self):\n",
    "        config = {\n",
    "            'fft_size': self.fft_size,\n",
    "            'hop_size': self.hop_size,\n",
    "            'n_mels': self.n_mels,\n",
    "            'sample_rate': self.sample_rate,\n",
    "            'f_min': self.f_min,\n",
    "            'f_max': self.f_max,\n",
    "        }\n",
    "        config.update(super(LogMelSpectrogram, self).get_config())\n",
    "\n",
    "        return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "808729a7-1356-4afe-b9f7-72f529a2ae1e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input (InputLayer)          [(None, 132300)]          0         \n",
      "                                                                 \n",
      " log_mel_spectrogram (LogMel  (None, 2052, 80, 1)      0         \n",
      " Spectrogram)                                                    \n",
      "                                                                 \n",
      " resizing (Resizing)         (None, 160, 80, 1)        0         \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 160, 80, 1)       320       \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " gaussian_noise (GaussianNoi  (None, 160, 80, 1)       0         \n",
      " se)                                                             \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 158, 78, 32)       320       \n",
      "                                                                 \n",
      " conv2d_1 (Conv2D)           (None, 156, 76, 64)       18496     \n",
      "                                                                 \n",
      " max_pooling2d (MaxPooling2D  (None, 78, 38, 64)       0         \n",
      " )                                                               \n",
      "                                                                 \n",
      " spatial_dropout2d (SpatialD  (None, 78, 38, 64)       0         \n",
      " ropout2D)                                                       \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 189696)            0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 128)               24281216  \n",
      "                                                                 \n",
      " gaussian_dropout (GaussianD  (None, 128)              0         \n",
      " ropout)                                                         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 5)                 645       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 24,300,997\n",
      "Trainable params: 24,300,837\n",
      "Non-trainable params: 160\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_shape = example_audio.shape[1:]\n",
    "input_length = len(input_shape)\n",
    "num_labels = len(label_names)\n",
    "\n",
    "frame_length=1024\n",
    "frame_step=64 # 128\n",
    "fft_length=None\n",
    "sample_rate=44100\n",
    "duration=3\n",
    "num_mel_channels=80\n",
    "freq_min=40 # 200\n",
    "freq_max=10000 # 8000\n",
    "    \n",
    "def ConvModel(n_classes=num_labels, sample_rate=sample_rate, duration=duration,\n",
    "              fft_size=frame_length, hop_size=frame_step, n_mels=num_mel_channels):\n",
    "    \n",
    "    n_samples = sample_rate * duration\n",
    "    \n",
    "    # Accept raw audio data as input\n",
    "    x = layers.Input(shape=(n_samples,), name='input', dtype='float32')\n",
    "    y = LogMelSpectrogram(sample_rate, fft_size, hop_size, n_mels, freq_min, freq_max)(x)\n",
    "    y = layers.Resizing(160,80)(y)\n",
    "    y = layers.BatchNormalization(axis=2)(y)\n",
    "    y = layers.GaussianNoise(1.5)(y)\n",
    "    y = layers.Conv2D(32, 3, activation='relu')(y)\n",
    "    y = layers.Conv2D(64, 3, activation='relu')(y)\n",
    "    y = layers.MaxPooling2D()(y)\n",
    "    y = layers.SpatialDropout2D(0.25)(y)\n",
    "    y = layers.Flatten()(y)\n",
    "    y = layers.Dense(128, activation='relu')(y)\n",
    "    y = layers.GaussianDropout(0.5)(y)\n",
    "    y = layers.Dense(num_labels)(y)\n",
    "    \n",
    "    return tf.keras.Model(inputs=x, outputs=y)\n",
    "\n",
    "model = ConvModel()\n",
    "\n",
    "model.summary()\n",
    "\n",
    "learning_rate = 0.001 # 0.0001\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(), # learning_rate=learning_rate\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    metrics=['accuracy'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cb42e65a-0658-4f63-87d5-cd4f6145600b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if cache_dir != '':\n",
    "    for f in Path(cache_dir).glob('*'):\n",
    "        try:\n",
    "            print(\"Removing cache file \"+str(f))\n",
    "            f.unlink()\n",
    "        except OSError as e:\n",
    "            print(\"Error: %s : %s\" % (f, e.strerror))\n",
    "    \n",
    "train_ds = train_ds.cache(cache_dir).shuffle(buffer_size=1000, reshuffle_each_iteration=True).prefetch(tf.data.AUTOTUNE)\n",
    "val_ds = val_ds.cache(cache_dir).prefetch(tf.data.AUTOTUNE)\n",
    "test_ds_precache = test_ds\n",
    "test_ds = test_ds.cache(cache_dir).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "761b0a0e-b911-4b67-8dcf-154a1f73a7b5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 1.5281 - accuracy: 0.3339\n",
      "Epoch 1: val_accuracy improved from -inf to 0.40885, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 146s 37ms/step - loss: 1.5280 - accuracy: 0.3339 - val_loss: 1.3425 - val_accuracy: 0.4088\n",
      "Epoch 2/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 1.3981 - accuracy: 0.3847\n",
      "Epoch 2: val_accuracy improved from 0.40885 to 0.44326, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 82s 26ms/step - loss: 1.3981 - accuracy: 0.3847 - val_loss: 1.3053 - val_accuracy: 0.4433\n",
      "Epoch 3/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 1.3460 - accuracy: 0.4189\n",
      "Epoch 3: val_accuracy improved from 0.44326 to 0.44911, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 1.3460 - accuracy: 0.4189 - val_loss: 1.2671 - val_accuracy: 0.4491\n",
      "Epoch 4/200\n",
      "3080/3082 [============================>.] - ETA: 0s - loss: 1.3020 - accuracy: 0.4407\n",
      "Epoch 4: val_accuracy improved from 0.44911 to 0.49310, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 1.3020 - accuracy: 0.4407 - val_loss: 1.1992 - val_accuracy: 0.4931\n",
      "Epoch 5/200\n",
      "3080/3082 [============================>.] - ETA: 0s - loss: 1.2569 - accuracy: 0.4661\n",
      "Epoch 5: val_accuracy improved from 0.49310 to 0.53369, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 1.2569 - accuracy: 0.4662 - val_loss: 1.1441 - val_accuracy: 0.5337\n",
      "Epoch 6/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 1.2113 - accuracy: 0.4899\n",
      "Epoch 6: val_accuracy improved from 0.53369 to 0.57532, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 1.2113 - accuracy: 0.4899 - val_loss: 1.0480 - val_accuracy: 0.5753\n",
      "Epoch 7/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 1.1642 - accuracy: 0.5155\n",
      "Epoch 7: val_accuracy improved from 0.57532 to 0.61485, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 82s 26ms/step - loss: 1.1642 - accuracy: 0.5155 - val_loss: 0.9948 - val_accuracy: 0.6149\n",
      "Epoch 8/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 1.1179 - accuracy: 0.5378\n",
      "Epoch 8: val_accuracy improved from 0.61485 to 0.64140, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 1.1179 - accuracy: 0.5378 - val_loss: 0.9808 - val_accuracy: 0.6414\n",
      "Epoch 9/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 1.0704 - accuracy: 0.5608\n",
      "Epoch 9: val_accuracy improved from 0.64140 to 0.68734, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 1.0704 - accuracy: 0.5608 - val_loss: 0.8769 - val_accuracy: 0.6873\n",
      "Epoch 10/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 1.0291 - accuracy: 0.5820\n",
      "Epoch 10: val_accuracy improved from 0.68734 to 0.70609, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 1.0291 - accuracy: 0.5820 - val_loss: 0.8324 - val_accuracy: 0.7061\n",
      "Epoch 11/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 0.9900 - accuracy: 0.6029\n",
      "Epoch 11: val_accuracy improved from 0.70609 to 0.72443, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 0.9900 - accuracy: 0.6029 - val_loss: 0.8010 - val_accuracy: 0.7244\n",
      "Epoch 12/200\n",
      "3080/3082 [============================>.] - ETA: 0s - loss: 0.9460 - accuracy: 0.6221\n",
      "Epoch 12: val_accuracy improved from 0.72443 to 0.74180, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 0.9463 - accuracy: 0.6220 - val_loss: 0.7362 - val_accuracy: 0.7418\n",
      "Epoch 13/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 0.9138 - accuracy: 0.6386\n",
      "Epoch 13: val_accuracy improved from 0.74180 to 0.75925, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 0.9138 - accuracy: 0.6386 - val_loss: 0.6991 - val_accuracy: 0.7593\n",
      "Epoch 14/200\n",
      "3080/3082 [============================>.] - ETA: 0s - loss: 0.8793 - accuracy: 0.6543\n",
      "Epoch 14: val_accuracy improved from 0.75925 to 0.76526, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 0.8794 - accuracy: 0.6543 - val_loss: 0.7022 - val_accuracy: 0.7653\n",
      "Epoch 15/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 0.8496 - accuracy: 0.6674\n",
      "Epoch 15: val_accuracy improved from 0.76526 to 0.78945, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 81s 26ms/step - loss: 0.8496 - accuracy: 0.6674 - val_loss: 0.6382 - val_accuracy: 0.7894\n",
      "Epoch 16/200\n",
      "3080/3082 [============================>.] - ETA: 0s - loss: 0.8179 - accuracy: 0.6846\n",
      "Epoch 16: val_accuracy improved from 0.78945 to 0.80187, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 77s 25ms/step - loss: 0.8180 - accuracy: 0.6846 - val_loss: 0.6351 - val_accuracy: 0.8019\n",
      "Epoch 17/200\n",
      "3082/3082 [==============================] - ETA: 0s - loss: 0.7961 - accuracy: 0.6960\n",
      "Epoch 17: val_accuracy improved from 0.80187 to 0.81599, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 0.7961 - accuracy: 0.6960 - val_loss: 0.5766 - val_accuracy: 0.8160\n",
      "Epoch 18/200\n",
      "3081/3082 [============================>.] - ETA: 0s - loss: 0.7675 - accuracy: 0.7068\n",
      "Epoch 18: val_accuracy improved from 0.81599 to 0.82070, saving model to Z:/AutoMusic/checkpoint\\automusic39.h5\n",
      "3082/3082 [==============================] - 80s 26ms/step - loss: 0.7674 - accuracy: 0.7068 - val_loss: 0.5953 - val_accuracy: 0.8207\n",
      "Epoch 19/200\n",
      " 581/3082 [====>.........................] - ETA: 1:01 - loss: 0.7481 - accuracy: 0.7168"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    save_weights_only=False,\n",
    "    monitor='val_accuracy',\n",
    "    mode='max',\n",
    "    save_best_only=True,\n",
    "    verbose=1)\n",
    "\n",
    "earlystop_callback = tf.keras.callbacks.EarlyStopping(\n",
    "    verbose=1,\n",
    "    patience=10,\n",
    "    min_delta=0.003,\n",
    "    restore_best_weights=True)\n",
    "\n",
    "# This may be a bad idea:\n",
    "# we're still retraining on ALL data, but this seems to speed up the epochs needed:\n",
    "# model.load_weights(checkpoint_filepath) # kick off training with the last run's weights\n",
    "\n",
    "history = model.fit(\n",
    "    train_ds,\n",
    "    validation_data=val_ds,\n",
    "    epochs=EPOCHS,\n",
    "    callbacks=[earlystop_callback,checkpoint_callback]\n",
    ")\n",
    "\n",
    "# shouldn't be needed with restore_best_weights=True\n",
    "# model.load_weights(checkpoint_filepath) # load the best saved weights... even if they aren't from this run, maybe?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c16258a-fd90-4e98-a181-cdb8078b107c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "metrics = history.history\n",
    "plt.figure(figsize=(16,6))\n",
    "plt.subplot(1,2,1)\n",
    "plt.plot(history.epoch, metrics['loss'], metrics['val_loss'])\n",
    "plt.legend(['loss', 'val_loss'])\n",
    "plt.ylim([0, max(plt.ylim())])\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss [CrossEntropy]')\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.plot(history.epoch, 100*np.array(metrics['accuracy']), 100*np.array(metrics['val_accuracy']))\n",
    "plt.legend(['accuracy', 'val_accuracy'])\n",
    "plt.ylim([0, 100])\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy [%]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15a37776-8f25-4432-a059-39ab95680916",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_true = tf.concat(list(test_ds_precache.map(lambda s,lab: lab)), axis=0)\n",
    "y_pred = model.predict(test_ds_precache)\n",
    "y_pred = y_pred.argmax(axis=1)\n",
    "\n",
    "confusion_mtx = tf.math.confusion_matrix(y_true, y_pred)\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.heatmap(confusion_mtx,\n",
    "            xticklabels=label_names,\n",
    "            yticklabels=label_names,\n",
    "            annot=True, fmt='g')\n",
    "plt.xlabel('Prediction')\n",
    "plt.ylabel('Label')\n",
    "plt.title('AutoMusic - Electronic Music Phrase Classifier')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55ebcff3-b636-4474-a160-718a0e72a4ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "x = test_file\n",
    "x = tf.io.read_file(str(x))\n",
    "x, sample_rate = tf.audio.decode_wav(x, desired_channels=1, desired_samples=44100*3)\n",
    "x = tf.squeeze(x, axis=-1)\n",
    "x = x[tf.newaxis,...]\n",
    "print(x.shape)\n",
    "\n",
    "prediction = model(x)\n",
    "plt.bar(label_names, tf.nn.softmax(prediction[0]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efd9706c-7cd1-459c-81f9-fa5aa53d718e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pyrekordbox\n",
    "from pyrekordbox.anlz import AnlzFile\n",
    "from pydub import AudioSegment\n",
    "from pydub.generators import WhiteNoise\n",
    "from pydub.effects import speedup, normalize\n",
    "import hashlib\n",
    "import os\n",
    "import random\n",
    "from pathlib import Path\n",
    "import logging\n",
    "l = logging.getLogger(\"pydub.converter\")\n",
    "l.setLevel(logging.CRITICAL)\n",
    "\n",
    "myfiles = pyrekordbox.anlz.walk_anlz_paths(pioneer_files)\n",
    "\n",
    "myfilecounter = 0\n",
    "\n",
    "with open(history_file,'w') as out:\n",
    "\n",
    "    for myfoundfile in myfiles:\n",
    "\n",
    "        try: mydat = AnlzFile.parse_file(myfoundfile[1]['DAT'])\n",
    "        except: continue\n",
    "        try: myext = AnlzFile.parse_file(myfoundfile[1]['EXT'])\n",
    "        except: continue\n",
    "\n",
    "        mymp3 = mydat.get('PPTH')\n",
    "        mymp3 = pioneer_prefix + mymp3\n",
    "\n",
    "        if \"Ultimate\" in str(mymp3):\n",
    "            # this is very specific to my library where there's some weird files full of random loops\n",
    "            continue\n",
    "\n",
    "        # print(mymp3)\n",
    "\n",
    "        mylabels = {}\n",
    "        mylabels['high'] = ['unknown', 'intro', 'up', 'down', 'unknown', 'chorus', 'outro', 'unknown', 'unknown', 'unknown', 'unknown']\n",
    "        mylabels['mid']  = ['unknown', 'intro', 'verse', 'verse', 'verse', 'verse', 'verse', 'verse', 'bridge', 'chorus', 'outro']\n",
    "        mylabels['low']  = mylabels['mid']\n",
    "\n",
    "        try: mytimecode = mydat.get('PQTZ')[2]\n",
    "        except: continue\n",
    "        try: mystructures = myext.get('PSSI').entries\n",
    "        except: continue\n",
    "        try: mymood = myext.get('PSSI').mood\n",
    "        except: continue\n",
    "\n",
    "        if (mymood != 1):\n",
    "            continue\n",
    "        else:\n",
    "            mymood = \"high\"\n",
    "\n",
    "        mytimecode = mydat.get('PQTZ')[2]\n",
    "\n",
    "        myfilelastbeat = myext.get('PSSI').end_beat\n",
    "\n",
    "        mysimplestructure = {}\n",
    "\n",
    "        myfilecounter += 1\n",
    "\n",
    "        for x in range(len(mystructures)):\n",
    "\n",
    "            mylabel = mystructures[x].kind\n",
    "            mylabel = mymood+\"_\"+mylabels[mymood][mylabel]\n",
    "\n",
    "            myfirstbeat = mystructures[x].beat\n",
    "\n",
    "            if (x+1<len(mystructures)):\n",
    "                mylastbeat = mystructures[x+1].beat\n",
    "            else:\n",
    "                mylastbeat = myfilelastbeat\n",
    "\n",
    "            if (mylastbeat+1<len(mytimecode)):\n",
    "                mylastbeat = mylastbeat\n",
    "            else:\n",
    "                mylastbeat = len(mytimecode)-1\n",
    "\n",
    "            mysegmentbars = (mylastbeat-myfirstbeat)//8\n",
    "            mysegmentleftovers = (mylastbeat-myfirstbeat)%8\n",
    "\n",
    "            myfirsttimecode = mytimecode[int(myfirstbeat)]*1000\n",
    "            mylasttimecode = mytimecode[int(mylastbeat)]*1000\n",
    "\n",
    "            mysimplestructure[int(myfirsttimecode)] = mylabel\n",
    "            mysimplestructure[int(mylasttimecode-1)] = mylabel\n",
    "\n",
    "        mysong = AudioSegment.from_file(mymp3,frame_rate=44100)\n",
    "        mono_audios = mysong.split_to_mono() \n",
    "        mysongmono = mono_audios[0]\n",
    "        # chomp output with a random start time of 0..2999ms into the file, to give some variations.\n",
    "        myrandomoffset = random.randint(0,2999)\n",
    "        # mysongmono = mysongmono.normalize()\n",
    "\n",
    "        mysongmono[myrandomoffset:].export(temp_wav_file,format=\"wav\")\n",
    "\n",
    "        x = tf.io.read_file(temp_wav_file)\n",
    "        x, sample_rate = tf.audio.decode_wav(x, desired_channels=1)\n",
    "        x = tf.squeeze(x, axis=-1)\n",
    "        waveform = x\n",
    "\n",
    "        slices = int(waveform.shape[0] / (44100*3))\n",
    "        samples = tf.split(waveform[: slices * (44100*3)], slices)\n",
    "\n",
    "        milliseconds = 0\n",
    "        right = 0\n",
    "        wrong = 0\n",
    "        transitions = 0\n",
    "\n",
    "        mylabelsseen = []\n",
    "        fileview = \"# \"\n",
    "\n",
    "        currentwrongs = 0\n",
    "\n",
    "        for sample in samples:\n",
    "\n",
    "            x = sample[tf.newaxis,...]\n",
    "\n",
    "            prediction = model(x)\n",
    "\n",
    "            # plt.bar(label_names, tf.nn.softmax(prediction[0]))\n",
    "            # plt.show()\n",
    "\n",
    "            # correct label for random offset above, because file may be skewed 1..2999ms ahead\n",
    "            #\n",
    "            res = mysimplestructure.get(milliseconds) or mysimplestructure[\n",
    "                  min(mysimplestructure.keys(), key = lambda key: abs(key-milliseconds-myrandomoffset))]\n",
    "\n",
    "            res2 = mysimplestructure.get(milliseconds) or mysimplestructure[\n",
    "                   min(mysimplestructure.keys(), key = lambda key: abs(key-milliseconds-myrandomoffset+3000))]\n",
    "\n",
    "            myaactualendlabel = str(res2)\n",
    "\n",
    "            mypredictedlabel = str(label_names[np.argmax(prediction)])\n",
    "            myactuallabel = str(res)\n",
    "\n",
    "            if myactuallabel != myaactualendlabel :\n",
    "\n",
    "                # print(\" ~~~ \"+mypredictedlabel+\" in transition from \"+myactuallabel+\" to \"+myaactualendlabel\n",
    "                print(\"~\", end=\"\")\n",
    "                print(\"~\", end=\"\",file=out)\n",
    "                transitions += 1\n",
    "                currentwrongs = 0\n",
    "\n",
    "            if mypredictedlabel == myactuallabel :\n",
    "\n",
    "                print(\"+\", end=\"\")\n",
    "                print(\"+\", end=\"\",file=out)\n",
    "                right += 1\n",
    "                currentwrongs = 0\n",
    "\n",
    "            else:\n",
    "\n",
    "                wrong += 1\n",
    "                myrandom = random.randint(0,9)\n",
    "\n",
    "                if (myactuallabel not in mylabelsseen or currentwrongs > 0 or myrandom == 0): # 10% chance of random sampling\n",
    "\n",
    "                    currentwrongs += 1\n",
    "\n",
    "                    if milliseconds+3001 < len(mysong):\n",
    "\n",
    "                        mylabelsseen.append(myactuallabel)\n",
    "                        mylabeldir = myactuallabel\n",
    "\n",
    "                        myoutputbasefile = myactuallabel+\".\"+Path(mymp3).stem+\"_\"+str(int(milliseconds)-myrandomoffset).rjust(6,'0')+\"_\"+str(int(milliseconds+3000)-myrandomoffset).rjust(6,'0')\n",
    "\n",
    "                        # myspeedup = mysong[milliseconds:milliseconds+3000].speedup(1.03)\n",
    "                        # mynoise = WhiteNoise().to_audio_segment(duration=len(myspeedup)).apply_gain(-20)\n",
    "                        # myspeedup_noise = myspeedup.overlay(mynoise)\n",
    "\n",
    "                        mytempsong = mysong[milliseconds:milliseconds+3000]\n",
    "                        # mynoise = WhiteNoise().to_audio_segment(duration=len(mytempsong)).apply_gain(-20)\n",
    "                        # mytempsong_noise = mytempsong.overlay(mynoise)\n",
    "\n",
    "                        mytempsong.export(data_dir+\"/\"+mylabeldir+\"/\"+myoutputbasefile+\"_FIXES.wav\", format=\"wav\")\n",
    "                        # myspeedup.export(data_dir+\"/\"+mylabeldir+\"/\"+myoutputbasefile+\"_speed_FIXES_N.wav\", format=\"wav\")\n",
    "                        # myspeedup_noise.export(data_dir+\"/\"+mylabeldir+\"/\"+myoutputbasefile+\"_speed_noise_FIXES_N.wav\", format=\"wav\")\n",
    "                        # mytempsong_noise.export(data_dir+\"/\"+mylabeldir+\"/\"+myoutputbasefile+\"_noise_FIXES_N.wav\", format=\"wav\")\n",
    "\n",
    "                        if myrandom == 0:\n",
    "                            print(\"R\", end=\"\")\n",
    "                            print(\"R\", end=\"\",file=out)\n",
    "                            currentwrongs = 0 # we only want one random sample, just to spice things up.\n",
    "                            # this also functions as a \"stop writing sequence of samples early\" randomness.\n",
    "                        else:\n",
    "                            print(\"W\", end=\"\")\n",
    "                            print(\"W\", end=\"\",file=out)\n",
    "\n",
    "                        if currentwrongs > 5: # stop over-feeding a LOT of wrong samples to the model next time. Let myrandom take care of this.\n",
    "                            currentwrongs = 0\n",
    "\n",
    "                    else:\n",
    "                        print(\"!\", end=\"\")\n",
    "                        print(\"!\", end=\"\",file=out)\n",
    "                else:\n",
    "                    print(\"!\", end=\"\")\n",
    "                    print(\"!\", end=\"\",file=out)\n",
    "\n",
    "            milliseconds += 3000\n",
    "\n",
    "        print(\" \")\n",
    "        print(\" \",file=out)\n",
    "        print(str(int(right/(right+wrong)*100))+\"% correct - \"+str(Path(mymp3).name)+\" (\"+str(myfilecounter)+\" of \"+str(mytotalfiles)+\")\")\n",
    "        print(str(int(right/(right+wrong)*100))+\"% correct - \"+str(Path(mymp3).name)+\" (\"+str(myfilecounter)+\" of \"+str(mytotalfiles)+\")\",file=out)\n",
    "        print(\" \")\n",
    "        print(\" \",file=out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee8b6390-7fc1-4f83-8dfb-94b93fdc2037",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "counter = 32\n",
    "histories = []\n",
    "\n",
    "while os.path.isfile(\"./Histories/history\"+str(counter)+\".txt\"):\n",
    "    \n",
    "    print(\"found history \"+str(counter))\n",
    "    histories.append(open(\"./Histories/history\"+str(counter)+\".txt\",'r').readlines())\n",
    "    counter += 1\n",
    "\n",
    "best = 0\n",
    "\n",
    "with open('./Histories/history_all.txt','w') as out:\n",
    "    \n",
    "    for z in range(len(histories[0])):\n",
    "        \n",
    "        for x in range(len(histories)):\n",
    "            if histories[x][z][0].isdigit():\n",
    "                testbest = int(histories[x][z].rstrip().partition(\"%\")[0])\n",
    "                if testbest > best:\n",
    "                    best = testbest\n",
    "        \n",
    "        for x in range(len(histories)):\n",
    "            if histories[x][z].rstrip() == \"\":\n",
    "                print(\"\",file=out)\n",
    "                break\n",
    "            else:\n",
    "                if histories[x][z][0].isdigit():\n",
    "                    testbest = int(histories[x][z].rstrip().partition(\"%\")[0])\n",
    "                    if best == testbest:\n",
    "                        print(\"> \",file=out,end='')\n",
    "                    else:\n",
    "                        print(\"  \",file=out,end='')\n",
    "                print(histories[x][z].rstrip(),file=out)\n",
    "                \n",
    "        best = 0\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}