scripts/2_CalculateFCMStatistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate FCM Statistics\n",
    "*Written by Viviane Clay*\n",
    "## Preparation\n",
    "### Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:39:50.135177Z",
     "start_time": "2021-01-08T08:39:46.181757Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np;\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import misc\n",
    "import matplotlib as mpl\n",
    "from itertools import combinations \n",
    "import pandas as pd\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data\n",
    "Get activations using the 'GetNetworkActivations.ipynb' script. Adapt paths as nescessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:49:38.080112Z",
     "start_time": "2021-01-08T08:49:37.688156Z"
    }
   },
   "outputs": [],
   "source": [
    "path = './Results/TowerTraining/BalancedTestSet/'\n",
    "figurePath = './Results/TowerTraining/Figures/AgentRewardComparisonsAdaTH/ActivationPatternsNormx2No-01-FlatC-50-50Test/XX'\n",
    "\n",
    "encExt = np.load(path + 'encExt.npy')\n",
    "encExtInt = np.load(path + 'encExtInt.npy')\n",
    "encInt = np.load(path + 'encInt.npy')\n",
    "encAE = np.load(path + 'encAE.npy')\n",
    "encC = np.load(path + 'encC.npy')\n",
    "ClassOut = np.load(path + 'outClass.npy')\n",
    "y = np.load(path + 'labels.npy')\n",
    "obs = np.load(path + 'obs.npy')\n",
    "# generate random activations\n",
    "randAct = np.random.randint(0,2,encC.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply Adaptive Threshold to Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:49:39.225786Z",
     "start_time": "2021-01-08T08:49:39.177916Z"
    }
   },
   "outputs": [],
   "source": [
    "c1ActTH = np.mean(encInt, axis = 0)\n",
    "c2ActTH = np.mean(encExt, axis = 0)\n",
    "c3ActTH = np.mean(encExtInt, axis = 0)\n",
    "AEActTH = np.mean(encAE, axis = 0)\n",
    "CActTH = np.mean(encC, axis = 0)\n",
    "\n",
    "encInt = np.array(np.abs(encInt) > np.abs(c1ActTH),dtype=int)\n",
    "encExt = np.array(np.abs(encExt) > np.abs(c2ActTH),dtype=int)\n",
    "encExtInt = np.array(np.abs(encExtInt) > np.abs(c3ActTH),dtype=int)\n",
    "encAE = np.array(np.abs(encAE) > np.abs(AEActTH),dtype=int)\n",
    "encC = np.array(np.abs(encC) > np.abs(CActTH),dtype=int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:57:18.315269Z",
     "start_time": "2021-01-08T08:57:18.126773Z"
    }
   },
   "outputs": [],
   "source": [
    "def getEdges(encodings):\n",
    "    TH = 0.5\n",
    "    comb = combinations(np.linspace(0,encodings.shape[1]-1,encodings.shape[1]), 2)\n",
    "    c = list(comb)\n",
    "    edges = np.zeros(len(c))  \n",
    "\n",
    "    for i in range(encodings.shape[0]):\n",
    "        for n in range(len(c)):\n",
    "            a = int(c[n][0])\n",
    "            b = int(c[n][1])\n",
    "            if (encodings[i][a]>TH) & (encodings[i][b]>TH):\n",
    "                edges[n] = edges[n]+1\n",
    "    return edges,c\n",
    "\n",
    "def learnConcept(examples, name, pattern_complexity,normalize = False):\n",
    "    e,c = getEdges(examples)\n",
    "    strongest = e.argsort()[-pattern_complexity:][::-1]\n",
    "    strongest_combs = []\n",
    "    strengths = []\n",
    "    #print('Combinations:      Strength:')\n",
    "    for s in strongest:\n",
    "        #print(str(c[s]) + '        ' + str(int(e[s])))\n",
    "        strongest_combs.append(c[s])\n",
    "        strengths.append(int(e[s]))\n",
    "    if normalize:\n",
    "        strengths = strengths/np.sum(strengths)\n",
    "    else:\n",
    "        strengths = np.array(strengths)/examples.shape[0]\n",
    "    return strongest_combs, strengths\n",
    "\n",
    "\n",
    "def getActsLogic(neuronCombs, strengths, data, class_TH):\n",
    "    patternOn = np.zeros(data.shape[0])\n",
    "    for i,n in enumerate(neuronCombs):\n",
    "        n1_act = (data[:,int(n[0])] > 0.5)\n",
    "        n2_act = (data[:,int(n[1])] > 0.5)\n",
    "        n_act = np.array(n1_act & n2_act, dtype=int) * strengths[i]\n",
    "        patternOn = patternOn + n_act\n",
    "    patternOn = (patternOn >= class_TH)\n",
    "    return patternOn\n",
    "\n",
    "def plotPatternActsLogic(patternOn, labels):\n",
    "    \n",
    "    plt.figure(figsize=(15,5))\n",
    "    plt.subplot(1,2,1)\n",
    "    plt.plot(patternOn)\n",
    "    plt.plot(labels/2)\n",
    "    plt.xlim([0,labels.shape[0]])\n",
    "    plt.ylim([0,1.1])\n",
    "    plt.legend(['Pattern On', 'Label Present'])\n",
    "    plt.yticks([])\n",
    "    plt.xlabel('Frames')\n",
    "    \n",
    "    plt.subplot(1,2,2)\n",
    "    c = plt.hist(patternOn + labels/2, bins = [0,0.5,1,1.5,2])\n",
    "    plt.xticks([0.25,0.75,1.25,1.75],['TN', 'FN', 'FP','TP'])\n",
    "    correct = np.round((c[0][0] + c[0][3])/labels.shape[0]*100,2)\n",
    "    precision = np.round(c[0][3]/(c[0][3]+c[0][2])*100,2)# If L is predicted it is correct X% of time\n",
    "    recall = np.round(c[0][3]/(c[0][3]+c[0][1])*100,2)# Correctly identifies X% of L\n",
    "    plt.title('Correct: ' + str(correct) + '% Precision: ' + str(precision) + '% Recall: ' + str(recall) + '%')\n",
    "    plt.show()\n",
    "    \n",
    "def plotCPR(combs, weights, enc, labels, THs):\n",
    "    cpr = np.zeros((THs.shape[0],3))\n",
    "    for i,TH in enumerate(THs):\n",
    "        patternOn = getActsLogic(combs, weights, enc, class_TH=TH)\n",
    "        c = plt.hist(patternOn + labels/2, bins = [0,0.5,1,1.5,2])\n",
    "        \n",
    "        correct = np.round((c[0][0] + c[0][3])/labels.shape[0]*100,2)\n",
    "        precision = np.round(c[0][3]/(c[0][3]+c[0][2])*100,2)# If L is predicted it is correct X% of time\n",
    "        recall = np.round(c[0][3]/(c[0][3]+c[0][1])*100,2)# Correctly identifies X% of L\n",
    "        \n",
    "        cpr[i] = np.array([correct,precision,recall])\n",
    "    plt.clf()\n",
    "    plt.figure(figsize=(15,5))\n",
    "    plt.plot(cpr)\n",
    "    plt.legend(['% Correct', 'Precision', 'Recall'])\n",
    "    plt.xticks(np.linspace(0,THs.shape[0]-1,THs.shape[0]),np.round(THs,2))\n",
    "    plt.xlim([0,THs.shape[0]-1])\n",
    "    plt.ylim([0,100])\n",
    "    plt.xlabel('Threshold')\n",
    "    plt.ylabel('Performance [Accuracy in %]')\n",
    "    plt.show()\n",
    "    \n",
    "def getCPR(combs, weights, enc, labels, THs,pattern):\n",
    "    cpr = np.zeros((THs.shape[0],5))\n",
    "    for i,TH in enumerate(THs):\n",
    "        if pattern=='single':\n",
    "            patternOn = getSingleActsLogic(combs, weights, enc, class_TH=TH)\n",
    "        elif pattern=='double':\n",
    "            patternOn = getActsLogic(combs, weights, enc, class_TH=TH)\n",
    "        elif pattern=='triple':\n",
    "            patternOn = getTripletsActsLogic(combs, weights, enc, class_TH=TH)\n",
    "        else:\n",
    "            print('unknown pattern identifier.')\n",
    "            break\n",
    "        c = plt.hist(patternOn + labels/2, bins = [0,0.5,1,1.5,2])\n",
    "        \n",
    "        correct = np.round((c[0][0] + c[0][3])/labels.shape[0]*100,2)\n",
    "        precision = np.round(c[0][3]/(c[0][3]+c[0][2])*100,2)# If L is predicted it is correct X% of time\n",
    "        recall = np.round(c[0][3]/(c[0][3]+c[0][1])*100,2)# Correctly identifies X% of L\n",
    "        f1score = np.round(2*(precision*recall/(precision+recall)),2)\n",
    "        \n",
    "        cpr[i] = np.array([correct,precision,recall,correct+precision+recall,f1score])\n",
    "    plt.close()\n",
    "    return np.nan_to_num(cpr)\n",
    "\n",
    "def getConceptSubset(AllConceptExamples, numExp):\n",
    "    subset = np.random.randint(0,AllConceptExamples.shape[0],numExp)\n",
    "    trainExp = AllConceptExamples[subset]\n",
    "    mask = np.ones(AllConceptExamples.shape, bool)\n",
    "    mask[subset] = False\n",
    "    testExp = AllConceptExamples[mask]\n",
    "    return trainExp,  testExp\n",
    "\n",
    "def getFP_FN(o,l):\n",
    "    w = np.where(l!=o)\n",
    "    FN = np.sum((o[w]==0)&(l[w]==1))/len(w[0])*100\n",
    "    FP = np.sum((o[w]>0)&(l[w]==0))/len(w[0])*100\n",
    "    return np.round(FP,3),np.round(FN,3)\n",
    "\n",
    "def getAccPR(o,l):\n",
    "    w = np.where(l!=o)\n",
    "    trueP = np.sum((o==l)&(o==1))\n",
    "    trueN = np.sum((o==l)&(o==0))\n",
    "    falseP = np.sum((o[w]>0)&(l[w]==0))\n",
    "    falseN = np.sum((o[w]==0)&(l[w]==1))\n",
    "    Acc = (trueP+trueN)/len(o)*100\n",
    "    Precision = trueP/(trueP+falseP)*100\n",
    "    Recall = trueP/(trueP+falseN)*100\n",
    "    return np.round(Acc,3),np.round(Precision,3),np.round(Recall,3),[trueP,trueN,falseP,falseN]\n",
    "\n",
    "def printStats(out, labels):\n",
    "    print('Accuracy: '+str(np.mean(out == labels)*100)+'%')\n",
    "    print('Everything correct in '+str(np.sum(np.mean(out == labels,axis=1)==1.0)/len(labels)*100)+'%')\n",
    "    print('Correct Door in '+str(np.round(np.mean(np.array(labels)[:,0]==out[:,0])*100,3))+'%')\n",
    "\n",
    "    AccND,PND,RND,NDA = getAccPR((out[:,0]==0).astype(int),(labels[:,0]==0).astype(int))\n",
    "    print('     No Door - Acc: '+str(AccND)+'% Precision: '+str(PND)+'% Recall: '+str(RND)+'%  |  TP,TN,FP,FN: '+str(NDA))\n",
    "\n",
    "    AccLD,PLD,RLD,LDA = getAccPR((out[:,0]==1).astype(int),(labels[:,0]==1).astype(int))\n",
    "    print('     Level Door - Acc: '+str(AccLD)+'% Precision: '+str(PLD)+'% Recall: '+str(RLD)+'%  |  TP,TN,FP,FN: '+str(LDA))\n",
    "\n",
    "    AccGD,PGD,RGD,GDA = getAccPR((out[:,0]==2).astype(int),(labels[:,0]==2).astype(int))\n",
    "    print('     Green Door - Acc: '+str(AccGD)+'% Precision: '+str(PGD)+'% Recall: '+str(RGD)+'%  |  TP,TN,FP,FN: '+str(GDA))\n",
    "\n",
    "    AccKD,PKD,RKD,KDA = getAccPR((out[:,0]==3).astype(int),(labels[:,0]==3).astype(int))\n",
    "    print('     Key Door - Acc: '+str(AccKD)+'% Precision: '+str(PKD)+'% Recall: '+str(RKD)+'%  |  TP,TN,FP,FN: '+str(KDA))\n",
    "\n",
    "    AccOD,POD,ROD,ODA = getAccPR((out[:,0]==4).astype(int),(labels[:,0]==4).astype(int))\n",
    "    print('     Other Door - Acc: '+str(AccOD)+'% Precision: '+str(POD)+'% Recall: '+str(ROD)+'%  |  TP,TN,FP,FN: '+str(ODA))\n",
    "\n",
    "    AccK,PK,RK,KA = getAccPR(out[:,1],np.array(labels)[:,1])\n",
    "    print('Correct Key in '+str(AccK)+'% - Precision: '+\n",
    "          str(PK)+'% Recall: '+str(RK)+'%  |  TP,TN,FP,FN: '+str(KA))\n",
    "\n",
    "    AccO,PO,RO,OA = getAccPR(out[:,2],np.array(labels)[:,2])\n",
    "    print('Correct Orb in '+str(AccO)+'% - Precision: '+\n",
    "          str(PO)+'% Recall: '+str(RO)+'%  |  TP,TN,FP,FN: '+str(OA))\n",
    "    \n",
    "def getStats(out, labels):\n",
    "\n",
    "    AccND,PND,RND,NDA = getAccPR((out[:,0]==0).astype(int),(labels[:,0]==0).astype(int))\n",
    "    \n",
    "    AccLD,PLD,RLD,LDA = getAccPR((out[:,0]==1).astype(int),(labels[:,0]==1).astype(int))\n",
    "\n",
    "    AccGD,PGD,RGD,GDA = getAccPR((out[:,0]==2).astype(int),(labels[:,0]==2).astype(int))\n",
    "\n",
    "    AccKD,PKD,RKD,KDA = getAccPR((out[:,0]==3).astype(int),(labels[:,0]==3).astype(int))\n",
    "\n",
    "    AccOD,POD,ROD,ODA = getAccPR((out[:,0]==4).astype(int),(labels[:,0]==4).astype(int))\n",
    "\n",
    "    AccK,PK,RK,KA = getAccPR(out[:,1],np.array(labels)[:,1])\n",
    "\n",
    "    AccO,PO,RO,OA = getAccPR(out[:,2],np.array(labels)[:,2])\n",
    "    \n",
    "    precisions = np.nan_to_num(np.array([PND, PLD, PGD, PKD, POD, PK, PO]))\n",
    "    recalls = np.nan_to_num(np.array([RND, RLD, RGD, RKD, ROD, RK, RO]))\n",
    "    f1scores = np.nan_to_num(np.round(2*(precisions*recalls/(precisions+recalls)),2))\n",
    "    \n",
    "    stats = {'Accuracy': np.mean(out == labels)*100,\n",
    "            'Accuracies': np.array([AccND, AccLD, AccGD, AccKD, AccOD, AccK, AccO]),\n",
    "            'Precisions': precisions,\n",
    "            'Recalls': recalls,\n",
    "            'F1Scores': f1scores}\n",
    "    return stats\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning Concepts\n",
    "#### Standard (Doubles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:42:21.399997Z",
     "start_time": "2021-01-08T08:42:21.228455Z"
    }
   },
   "outputs": [],
   "source": [
    "def LearnAllConcepts(numExp, pattern_complexity, encoding, to_learn, concept_def):\n",
    "    allConcepts = []\n",
    "        \n",
    "    for i, concept in enumerate(to_learn):\n",
    "        learnExamples, testExamples = getConceptSubset(np.where(concept_def[i])[0], numExp)\n",
    "\n",
    "        examples = encoding[learnExamples]\n",
    "        labels_test = concept_def[i]\n",
    "\n",
    "        combs, weights = learnConcept(examples, 'no door', pattern_complexity=pattern_complexity,normalize=True)\n",
    "        \n",
    "        true_exp = np.where(labels_test==True)[0]\n",
    "        false_exp = np.where(labels_test==False)[0]\n",
    "        np.random.shuffle(false_exp)\n",
    "        balanced_exp = np.append(true_exp,false_exp[:true_exp.shape[0]])\n",
    "\n",
    "        cpr_train = getCPR(combs, weights, examples, np.ones(examples.shape[0]), np.linspace(0,1,21),pattern='double')\n",
    "        cpr_test = getCPR(combs, weights, encoding[balanced_exp], labels_test[balanced_exp], np.linspace(0,1,21),pattern='double')\n",
    "\n",
    "        ConceptStats = {'Concept':concept,\n",
    "                     'Examples': learnExamples,\n",
    "                   'TestExamples': testExamples,\n",
    "                     'ExamplesEnc': examples,\n",
    "                     'Combinations': combs,\n",
    "                     'Weights': weights,\n",
    "                     'CPR_Train': cpr_train,\n",
    "                     'CPR_Test': cpr_test}\n",
    "\n",
    "        allConcepts.append(ConceptStats)\n",
    "    return allConcepts\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Single Concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learnSingleConcept(examples, pattern_complexity,normalize = False):\n",
    "    neuronActs = np.sum(examples,axis=0)\n",
    "    strongest = neuronActs.argsort()[-pattern_complexity:][::-1]\n",
    "    strongest_N = []\n",
    "    strengths = []\n",
    "    for s in strongest:\n",
    "        #print(str(s) + '        ' + str(int(neuronActs[s])))\n",
    "        strongest_N.append(s)\n",
    "        strengths.append(int(neuronActs[s]))#/examples.shape[0])\n",
    "    if normalize:\n",
    "        strengths = strengths/np.sum(strengths)\n",
    "    else:\n",
    "        strengths = np.array(strengths)/examples.shape[0]\n",
    "    return strongest_N, strengths\n",
    "\n",
    "def getSingleActsLogic(neurons, strengths, data, class_TH):\n",
    "    #strengths = strengths/np.max(strengths)\n",
    "    patternOn = np.zeros(data.shape[0])\n",
    "    for i,n in enumerate(neurons):\n",
    "        n1_act = (data[:,int(n)] > 0.5)\n",
    "        n_act = np.array(n1_act, dtype=int) * strengths[i]\n",
    "        #print(n_act)\n",
    "        patternOn = patternOn + n_act\n",
    "    '''print('pattern on: ' + str(patternOn))\n",
    "    print('Max: ' + str(np.max(patternOn))[:4])\n",
    "    print('Min: ' + str(np.min(patternOn)))\n",
    "    print('Mean: ' + str(np.mean(patternOn)))\n",
    "    print('Median: ' + str(np.median(patternOn)))'''\n",
    "    #plt.hist(patternOn)\n",
    "    patternOn = (patternOn >= class_TH)#np.array(neuronCombs).shape[0]-1)\n",
    "    return patternOn\n",
    "\n",
    "def LearnAllSingleConcepts(numExp, pattern_complexity, encoding, to_learn, concept_def):\n",
    "    allConcepts = []\n",
    "        \n",
    "    for i, concept in enumerate(to_learn):\n",
    "        learnExamples, testExamples = getConceptSubset(np.where(concept_def[i])[0], numExp)\n",
    "\n",
    "        examples = encoding[learnExamples]\n",
    "        labels_test = concept_def[i]\n",
    "\n",
    "        combs, weights = learnSingleConcept(examples, pattern_complexity=pattern_complexity,normalize=True)\n",
    "        \n",
    "        true_exp = np.where(labels_test==True)[0]\n",
    "        false_exp = np.where(labels_test==False)[0]\n",
    "        np.random.shuffle(false_exp)\n",
    "        balanced_exp = np.append(true_exp,false_exp[:true_exp.shape[0]])\n",
    "\n",
    "        cpr_train = getCPR(combs, weights, examples, np.ones(examples.shape[0]), np.linspace(0,1,21),pattern='single')\n",
    "        cpr_test = getCPR(combs, weights, encoding[balanced_exp], labels_test[balanced_exp], np.linspace(0,1,21),pattern='single')\n",
    "        \n",
    "        ConceptStats = {'Concept':concept,\n",
    "                     'Examples': learnExamples,\n",
    "                   'TestExamples': testExamples,\n",
    "                     'ExamplesEnc': examples,\n",
    "                     'Combinations': combs,\n",
    "                     'Weights': weights,\n",
    "                     'CPR_Train': cpr_train,\n",
    "                     'CPR_Test': cpr_test}\n",
    "\n",
    "        allConcepts.append(ConceptStats)\n",
    "    return allConcepts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Triplet Concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTriplets(encodings):\n",
    "    TH = 0.5\n",
    "    comb = combinations(np.linspace(0,encodings.shape[1]-1,encodings.shape[1]), 3)\n",
    "    c = list(comb)\n",
    "    triplets = np.zeros(len(c))\n",
    "\n",
    "    for i in range(encodings.shape[0]):\n",
    "        for n in range(len(c)):\n",
    "            Na = int(c[n][0])\n",
    "            Nb = int(c[n][1])\n",
    "            Nc = int(c[n][2])\n",
    "            if (encodings[i][Na]>TH) & (encodings[i][Nb]>TH) & (encodings[i][Nc]>TH):\n",
    "                triplets[n] = triplets[n]+1\n",
    "    return triplets,c\n",
    "\n",
    "def learnTripletConcept(examples, pattern_complexity,normalize = False):\n",
    "    t,c = getTriplets(examples)\n",
    "    strongest = t.argsort()[-pattern_complexity:][::-1]\n",
    "    strongest_combs = []\n",
    "    strengths = []\n",
    "    #print('Combinations:      Strength:')\n",
    "    for s in strongest:\n",
    "        #print(str(c[s]) + '        ' + str(int(t[s])))\n",
    "        strongest_combs.append(c[s])\n",
    "        strengths.append(int(t[s]))\n",
    "    if normalize:\n",
    "        strengths = strengths/np.sum(strengths)\n",
    "    else:\n",
    "        strengths = np.array(strengths)/examples.shape[0]\n",
    "    return strongest_combs, strengths\n",
    "\n",
    "\n",
    "def getTripletsActsLogic(neuronCombs, strengths, data, class_TH):\n",
    "    patternOn = np.zeros(data.shape[0])\n",
    "    for i,n in enumerate(neuronCombs):\n",
    "        n1_act = (data[:,int(n[0])] > 0.5)\n",
    "        n2_act = (data[:,int(n[1])] > 0.5)\n",
    "        n3_act = (data[:,int(n[2])] > 0.5)\n",
    "        n_act = np.array(n1_act & n2_act & n3_act, dtype=int) * strengths[i]\n",
    "        patternOn = patternOn + n_act\n",
    "    patternOn = (patternOn >= class_TH)\n",
    "    return patternOn\n",
    "\n",
    "def LearnAllTripletConcepts(numExp, pattern_complexity, encoding, to_learn, concept_def):\n",
    "    allConcepts = []\n",
    "        \n",
    "    for i, concept in enumerate(to_learn):\n",
    "        learnExamples, testExamples = getConceptSubset(np.where(concept_def[i])[0], numExp)\n",
    "\n",
    "        examples = encoding[learnExamples]\n",
    "        labels_test = concept_def[i]\n",
    "\n",
    "        combs, weights = learnTripletConcept(examples, pattern_complexity=pattern_complexity,normalize=True)\n",
    "        \n",
    "        true_exp = np.where(labels_test==True)[0]\n",
    "        false_exp = np.where(labels_test==False)[0]\n",
    "        np.random.shuffle(false_exp)\n",
    "        balanced_exp = np.append(true_exp,false_exp[:true_exp.shape[0]])\n",
    "\n",
    "        cpr_train = getCPR(combs, weights, examples, np.ones(examples.shape[0]), np.linspace(0,1,21),pattern='triple')\n",
    "        cpr_test = getCPR(combs, weights, encoding[balanced_exp], labels_test[balanced_exp], np.linspace(0,1,21),pattern='triple')\n",
    "\n",
    "        ConceptStats = {'Concept':concept,\n",
    "                     'Examples': learnExamples,\n",
    "                     'Combinations': combs,\n",
    "                     'Weights': weights,\n",
    "                     'CPR_Train': cpr_train,\n",
    "                     'CPR_Test': cpr_test}\n",
    "\n",
    "        allConcepts.append(ConceptStats)\n",
    "    return allConcepts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCPR_SVM(classification, labels, THs, single=False):\n",
    "    c = plt.hist(classification + labels/2, bins = [0,0.5,1,1.5,2])\n",
    "        \n",
    "    correct = np.round((c[0][0] + c[0][3])/labels.shape[0]*100,2)\n",
    "    precision = np.round(c[0][3]/(c[0][3]+c[0][2])*100,2)# If L is predicted it is correct X% of time\n",
    "    recall = np.round(c[0][3]/(c[0][3]+c[0][1])*100,2)# Correctly identifies X% of L\n",
    "    f1score = np.round(2*(precision*recall/(precision+recall)),2)\n",
    "        \n",
    "    cpr = np.array([correct,precision,recall,correct+precision+recall,f1score])\n",
    "    plt.clf()\n",
    "    return np.nan_to_num(cpr)\n",
    "\n",
    "def LearnAllConceptsSVM(numExp, encoding, to_learn, concept_def):\n",
    "    allConcepts = []\n",
    "        \n",
    "    for i, concept in enumerate(to_learn):\n",
    "        learnExamples, testExamples = getConceptSubset(np.where(concept_def[i])[0], numExp)\n",
    "        learnExamplesNeg, _ = getConceptSubset(np.where(np.logical_not(concept_def[i]))[0], numExp)\n",
    "\n",
    "        examples = encoding[learnExamples]\n",
    "        examplesNeg = encoding[learnExamplesNeg]\n",
    "        labels_test = concept_def[i]\n",
    "\n",
    "        clf_linear = svm.SVC(kernel='linear')\n",
    "        clf_linear.fit(np.concatenate([examples,examplesNeg]), np.concatenate([np.ones(examples.shape[0]),np.zeros(examplesNeg.shape[0])]))\n",
    "        out_linear = clf_linear.predict(encInt)\n",
    "        \n",
    "        true_exp = np.where(labels_test==True)[0]\n",
    "        false_exp = np.where(labels_test==False)[0]\n",
    "        np.random.shuffle(false_exp)\n",
    "        balanced_exp = np.append(true_exp,false_exp[:true_exp.shape[0]])\n",
    "\n",
    "        cpr_test = getCPR_SVM(out_linear[balanced_exp], labels_test[balanced_exp], np.linspace(0,1,21))\n",
    "\n",
    "        ConceptStats = {'Concept':concept,\n",
    "                     'Examples': learnExamples,\n",
    "                     'CPR_Test': cpr_test}\n",
    "\n",
    "        allConcepts.append(ConceptStats)\n",
    "    return allConcepts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FCM\n",
    "### Define Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T08:43:07.869435Z",
     "start_time": "2021-01-08T08:43:07.850487Z"
    }
   },
   "outputs": [],
   "source": [
    "to_learn = ['Level Door', 'Green Door', 'Key Door', 'Other Door', 'Key', 'Orb']#'No Door', \n",
    "concept_def = [y[:,0]==1, y[:,0]==2, y[:,0]==3, y[:,0]==4, y[:,1]==1, y[:,2]==1]#y[:,0]==0, \n",
    "numExp = 5\n",
    "pattern_complexity = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allConceptsSingle = LearnAllSingleConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "allConceptsTriplet = LearnAllTripletConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "allConceptsSVM = LearnAllConceptsSVM(numExp, encInt, to_learn, concept_def)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM on 4 Network Activations\n",
    "Comparing FCM on activations of agent, autoencoder, classifier and random."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T09:03:58.976387Z",
     "start_time": "2021-01-08T09:03:15.456353Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\vkakerbeck\\miniconda3\\lib\\site-packages\\ipykernel_launcher.py:100: RuntimeWarning: invalid value encountered in double_scalars\n",
      "c:\\users\\vkakerbeck\\miniconda3\\lib\\site-packages\\ipykernel_launcher.py:102: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsAECR_TH2050 = pd.DataFrame(columns = {'Concept','Encoding', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "\n",
    "for pattern_complexity in [pattern_complexity]:\n",
    "    for numExp in [numExp]:\n",
    "        for i in range(numRuns):\n",
    "            print(i)\n",
    "            allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            allConceptsAE = LearnAllConcepts(numExp, pattern_complexity, encAE, to_learn, concept_def)\n",
    "            allConceptsC = LearnAllConcepts(numExp, pattern_complexity, encC, to_learn, concept_def)\n",
    "            allConceptsR = LearnAllConcepts(numExp, pattern_complexity, randAct, to_learn, concept_def)\n",
    "            for c in allConcepts:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Agent',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Agent',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsAE:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Autoencoder',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Autoencoder',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "               \n",
    "            for c in allConceptsC:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Classifier',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Classifier',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "           \n",
    "            for c in allConceptsR:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Random',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsAECR_TH2050 = bestTHStatsAECR_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Random',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsAECR_TH2050.to_csv(figurePath+'bestTHStatsAECRNormx2_TH2050.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-08T09:06:14.658147Z",
     "start_time": "2021-01-08T09:06:14.610268Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Precision</th>\n",
       "      <th>TH</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Concept</th>\n",
       "      <th>NumExp</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>CPR</th>\n",
       "      <th>F1Score</th>\n",
       "      <th>Encoding</th>\n",
       "      <th>Examples</th>\n",
       "      <th>PC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78.77</td>\n",
       "      <td>0.2</td>\n",
       "      <td>46.0</td>\n",
       "      <td>Level Door</td>\n",
       "      <td>5</td>\n",
       "      <td>66.8</td>\n",
       "      <td>191.57</td>\n",
       "      <td>58.08</td>\n",
       "      <td>Agent</td>\n",
       "      <td>[379, 548, 571, 370, 452]</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93.10</td>\n",
       "      <td>0.5</td>\n",
       "      <td>32.4</td>\n",
       "      <td>Level Door</td>\n",
       "      <td>5</td>\n",
       "      <td>65.0</td>\n",
       "      <td>190.50</td>\n",
       "      <td>48.07</td>\n",
       "      <td>Agent</td>\n",
       "      <td>[379, 548, 571, 370, 452]</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>71.96</td>\n",
       "      <td>0.2</td>\n",
       "      <td>61.6</td>\n",
       "      <td>Green Door</td>\n",
       "      <td>5</td>\n",
       "      <td>68.8</td>\n",
       "      <td>202.36</td>\n",
       "      <td>66.38</td>\n",
       "      <td>Agent</td>\n",
       "      <td>[602, 605, 638, 622, 686]</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>83.33</td>\n",
       "      <td>0.5</td>\n",
       "      <td>44.0</td>\n",
       "      <td>Green Door</td>\n",
       "      <td>5</td>\n",
       "      <td>67.6</td>\n",
       "      <td>194.93</td>\n",
       "      <td>57.59</td>\n",
       "      <td>Agent</td>\n",
       "      <td>[602, 605, 638, 622, 686]</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65.09</td>\n",
       "      <td>0.2</td>\n",
       "      <td>27.6</td>\n",
       "      <td>Key Door</td>\n",
       "      <td>5</td>\n",
       "      <td>56.4</td>\n",
       "      <td>149.09</td>\n",
       "      <td>38.76</td>\n",
       "      <td>Agent</td>\n",
       "      <td>[1008, 1007, 1089, 1071, 871]</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Precision   TH  Recall     Concept NumExp  Accuracy     CPR  F1Score  \\\n",
       "0      78.77  0.2    46.0  Level Door      5      66.8  191.57    58.08   \n",
       "1      93.10  0.5    32.4  Level Door      5      65.0  190.50    48.07   \n",
       "2      71.96  0.2    61.6  Green Door      5      68.8  202.36    66.38   \n",
       "3      83.33  0.5    44.0  Green Door      5      67.6  194.93    57.59   \n",
       "4      65.09  0.2    27.6    Key Door      5      56.4  149.09    38.76   \n",
       "\n",
       "  Encoding                       Examples    PC  \n",
       "0    Agent      [379, 548, 571, 370, 452]  10.0  \n",
       "1    Agent      [379, 548, 571, 370, 452]  10.0  \n",
       "2    Agent      [602, 605, 638, 622, 686]  10.0  \n",
       "3    Agent      [602, 605, 638, 622, 686]  10.0  \n",
       "4    Agent  [1008, 1007, 1089, 1071, 871]  10.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bestTHStatsAECR_TH2050.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM on all Activations for Different Numbers of Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsAECR_numExp = pd.DataFrame(columns = {'Concept','Encoding','PC', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "\n",
    "to_learnD = [to_learn[1]]\n",
    "concept_defD = [concept_def[1]]\n",
    "for pattern_complexity in [pattern_complexity]:\n",
    "    for numExp in [1,2,3,4,5,10,15,20,50]:\n",
    "        for i in range(numRuns):\n",
    "            print(i)\n",
    "            allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learnD, concept_defD)\n",
    "            allConceptsAE = LearnAllConcepts(numExp, pattern_complexity, encAE, to_learnD, concept_defD)\n",
    "            allConceptsC = LearnAllConcepts(numExp, pattern_complexity, encC, to_learnD, concept_defD)\n",
    "            allConceptsR = LearnAllConcepts(numExp, pattern_complexity, randAct, to_learnD, concept_defD)\n",
    "            for c in allConcepts:\n",
    "                if c['Concept']=='Level Door':\n",
    "                    TH20 = c['CPR_Test'][4]\n",
    "                    bestTHStatsAECR_numExp = bestTHStatsAECR_numExp.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Agent',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsAE:\n",
    "                if c['Concept']=='Level Door':\n",
    "                    TH20 = c['CPR_Test'][4]\n",
    "                    bestTHStatsAECR_numExp = bestTHStatsAECR_numExp.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Autoencoder',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsC:\n",
    "                if c['Concept']=='Level Door':\n",
    "                    TH20 = c['CPR_Test'][4]\n",
    "                    bestTHStatsAECR_numExp = bestTHStatsAECR_numExp.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Classifier',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "            for c in allConceptsR:\n",
    "                if c['Concept']=='Level Door':\n",
    "                    TH20 = c['CPR_Test'][4]\n",
    "                    bestTHStatsAECR_numExp = bestTHStatsAECR_numExp.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Random',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsAECR_numExp.to_csv(figurePath+'bestTHStatsAECR_numExpLDoor_TH20.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM on all Activations for Different Pattern Complexities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsAECR_PCComp_AllC = pd.DataFrame(columns = {'Concept','Encoding','PC', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "\n",
    "for pattern_complexity in [1,2,3,4,5,10,15,20]:\n",
    "    for numExp in [numExp]:\n",
    "        for i in range(numRuns):\n",
    "            print(str(pattern_complexity) + ' - ' + str(i))\n",
    "            allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            allConceptsAE = LearnAllConcepts(numExp, pattern_complexity, encAE, to_learn, concept_def)\n",
    "            allConceptsC = LearnAllConcepts(numExp, pattern_complexity, encC, to_learn, concept_def)\n",
    "            allConceptsR = LearnAllConcepts(numExp, pattern_complexity, randAct, to_learn, concept_def)\n",
    "            for c in allConcepts:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                bestTHStatsAECR_PCComp_AllC = bestTHStatsAECR_PCComp_AllC.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Agent',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsAE:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                bestTHStatsAECR_PCComp_AllC = bestTHStatsAECR_PCComp_AllC.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Autoencoder',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsC:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                bestTHStatsAECR_PCComp_AllC = bestTHStatsAECR_PCComp_AllC.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Classifier',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "            for c in allConceptsR:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                bestTHStatsAECR_PCComp_AllC = bestTHStatsAECR_PCComp_AllC.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Random',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "    # intermediate saves in case of memory issues:\n",
    "    #bestTHStatsAECR_PCComp_AllC.to_csv(figurePath+'bestTHStatsAECR_PCComp_AllC_'+str(pattern_complexity)+'.csv')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsAECR_PCComp_AllC.to_csv(figurePath+'bestTHStatsAECR_PCComp_AllC.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM on all Activations for Different Thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsAECR_TH = pd.DataFrame(columns = {'Concept','Encoding','Pattern Complexity', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall'})\n",
    "\n",
    "for pattern_complexity in [pattern_complexity]:\n",
    "    for numExp in [numExp]:\n",
    "        for i in range(numRuns):\n",
    "            allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            allConceptsAE = LearnAllConcepts(numExp, pattern_complexity, encAE, to_learn, concept_def)\n",
    "            allConceptsC = LearnAllConcepts(numExp, pattern_complexity, encC, to_learn, concept_def)\n",
    "            allConceptsR = LearnAllConcepts(numExp, pattern_complexity, randAct, to_learn, concept_def)\n",
    "            for c in allConcepts:\n",
    "                for i,THTest in enumerate(c['CPR_Test']):\n",
    "                    bestTHStatsAECR_TH = bestTHStatsAECR_TH.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': np.round(np.linspace(0,1,21)[i],2),\n",
    "                                                  'Accuracy': THTest[0], 'Precision': THTest[1],\n",
    "                                                  'Recall': THTest[2]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsAE:\n",
    "                for i,THTest in enumerate(c['CPR_Test']):\n",
    "                    bestTHStatsAECR_TH = bestTHStatsAECR_TH.append({'Concept': c['Concept'],'Encoding':'Autoencoder','Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': np.round(np.linspace(0,1,21)[i],2),\n",
    "                                                  'Accuracy': THTest[0], 'Precision': THTest[1],\n",
    "                                                  'Recall': THTest[2]},ignore_index=True)\n",
    "                \n",
    "            for c in allConceptsC:\n",
    "                for i,THTest in enumerate(c['CPR_Test']):\n",
    "                    bestTHStatsAECR_TH = bestTHStatsAECR_TH.append({'Concept': c['Concept'],'Encoding':'Classifier','Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': np.round(np.linspace(0,1,21)[i],2),\n",
    "                                                  'Accuracy': THTest[0], 'Precision': THTest[1],\n",
    "                                                  'Recall': THTest[2]},ignore_index=True)\n",
    "            for c in allConceptsR:\n",
    "                for i,THTest in enumerate(c['CPR_Test']):\n",
    "                    bestTHStatsAECR_TH = bestTHStatsAECR_TH.append({'Concept': c['Concept'],'Encoding':'Random','Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': np.round(np.linspace(0,1,21)[i],2),\n",
    "                                                  'Accuracy': THTest[0], 'Precision': THTest[1],\n",
    "                                                  'Recall': THTest[2]},ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsAECR_TH.to_csv(figurePath+'bestTHStatsAECRNormx2_TH.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM on Different Agent Reward Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsA_TH2050 = pd.DataFrame(columns = {'Concept','Encoding','Pattern Complexity', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "for pattern_complexity in [pattern_complexity]:\n",
    "    for numExp in [numExp]:\n",
    "        for i in range(numRuns):\n",
    "            print(i)\n",
    "            allConceptsE = LearnAllConcepts(numExp, pattern_complexity, encExt, to_learn, concept_def)\n",
    "            allConceptsEI = LearnAllConcepts(numExp, pattern_complexity, encExtInt, to_learn, concept_def)\n",
    "            allConceptsI = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            \n",
    "            for c in allConceptsE:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'External',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'External',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)      \n",
    "            for c in allConceptsEI:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Ext+Int',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Ext+Int',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "            for c in allConceptsI:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Internal',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                \n",
    "                bestTHStatsA_TH2050 = bestTHStatsA_TH2050.append({'Concept': c['Concept'],\n",
    "                                                                            'Encoding':'Internal',\n",
    "                                                                            'PC':pattern_complexity,\n",
    "                                                                            'Examples': c['Examples'],\n",
    "                                                                            'NumExp':numExp,'TH': 0.5,\n",
    "                                                                            'Accuracy': TH50[0],\n",
    "                                                                            'Precision': TH50[1],\n",
    "                                                                            'Recall': TH50[2],\n",
    "                                                                            'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "                \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsA_TH2050.to_csv(figurePath+'bestTHStatsAgentsNormx2_TH2050.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM with Different Types of Patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsA_P = pd.DataFrame(columns = {'Concept','Encoding','Pattern','Pattern Complexity', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "\n",
    "for pattern_complexity in [pattern_complexity]:\n",
    "    for numExp in [numExp]:\n",
    "        for i in range(numRuns):\n",
    "            print(i)\n",
    "            allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            allConceptsSingle = LearnAllSingleConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            allConceptsTriplet = LearnAllTripletConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "            for c in allConceptsSingle:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Single',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                                            'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Single',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0.5,\n",
    "                                                  'Accuracy': TH50[0], 'Precision': TH50[1],\n",
    "                                                  'Recall': TH50[2],'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "                \n",
    "            for c in allConcepts:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Double',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                                            'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Double',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0.5,\n",
    "                                                  'Accuracy': TH50[0], 'Precision': TH50[1],\n",
    "                                                  'Recall': TH50[2],'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "            for c in allConceptsTriplet:\n",
    "                TH20 = c['CPR_Test'][4]\n",
    "                TH50 = c['CPR_Test'][10]\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Triplet',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                                            'TH': 0.2,\n",
    "                                                                            'Accuracy': TH20[0],\n",
    "                                                                            'Precision': TH20[1],\n",
    "                                                                            'Recall': TH20[2],\n",
    "                                                                            'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "                bestTHStatsA_P = bestTHStatsA_P.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Triplet',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0.5,\n",
    "                                                  'Accuracy': TH50[0], 'Precision': TH50[1],\n",
    "                                                  'Recall': TH50[2],'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsA_P.to_csv(figurePath+'bestTHStatsA_Pattern.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCM Compared to SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numRuns = 100\n",
    "bestTHStatsA_SVM = pd.DataFrame(columns = {'Concept','Encoding','Pattern','Pattern Complexity', 'Examples','NumExp','TH', \n",
    "                                        'Accuracy', 'Precision', 'Recall','CPR','F1Score'})\n",
    "\n",
    "for i in range(numRuns):\n",
    "    print(i)\n",
    "    allConcepts = LearnAllConcepts(numExp, pattern_complexity, encInt, to_learn, concept_def)\n",
    "    allConceptsSVM = LearnAllConceptsSVM(numExp, encInt, to_learn, concept_def)\n",
    "    for c in allConcepts:\n",
    "        TH20 = c['CPR_Test'][4]\n",
    "        TH50 = c['CPR_Test'][10]\n",
    "        bestTHStatsA_SVM = bestTHStatsA_SVM.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Double',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0.2,\n",
    "                                                  'Accuracy': TH20[0], 'Precision': TH20[1],\n",
    "                                                  'Recall': TH20[2],'CPR':TH20[3],'F1Score':TH20[4]},ignore_index=True)\n",
    "        bestTHStatsA_SVM = bestTHStatsA_SVM.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'Double',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0.5,\n",
    "                                                  'Accuracy': TH50[0], 'Precision': TH50[1],\n",
    "                                                  'Recall': TH50[2],'CPR':TH50[3],'F1Score':TH50[4]},ignore_index=True)\n",
    "    for c in allConceptsSVM:\n",
    "        bestTHStatsA_SVM = bestTHStatsA_SVM.append({'Concept': c['Concept'],'Encoding':'Agent','Pattern':'SVM',\n",
    "                                                              'Pattern Complexity':pattern_complexity, 'Examples': c['Examples'],'NumExp':numExp,\n",
    "                                                  'TH': 0,\n",
    "                                                  'Accuracy': c['CPR_Test'][0], 'Precision': c['CPR_Test'][1],\n",
    "                                                  'Recall': c['CPR_Test'][2],'CPR':c['CPR_Test'][3],'F1Score':c['CPR_Test'][4]},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestTHStatsA_SVM.to_csv(figurePath+'bestTHStatsA_SVM.csv')"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}