Recognizing Pokémons

June 16, 2018 - a pipeline from scrapping raw pokémon images to a pokémon recognition model with error analysis.

This is a pipeline from scraping pokémon images to building a predictive model to recognize them.


Imports

Cloud Code

In [ ]:
# Paperspace Commands
!pip3 install -U -q tensorflow
!pip3 install -U -q keras
!pip3 install -U -q numpy
!pip3 install -U -q pandas
!pip3 install -U -q opencv-python
!pip3 install -U -q h5py
!pip3 install -U -q lxml
#!unzip pokemons/images.zip -d pokemons
In [ ]:
!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
In [6]:
def downloadFileCloud(id, name):
    """ Downloads file from Google Drive. """
    if not name in os.listdir():
        file = drive.CreateFile({'id': id})
        file.GetContentFile(name)

def downloadFolderCloud(id, extension):
    """ Downloads folder with contents from Google Drive. """
    files = drive.ListFile({'q': id + " in parents and trashed=false"}).GetList()
    numFiles = len(files)
    presentFiles = [f for f in os.listdir("./") if f.endswith(extension)]
    for i, file in enumerate(files):
        filename = file.get('title')
        if filename not in presentFiles: file.GetContentFile(filename)
        if i%100==0 and i!=0: print("{} of {} files downloaded.".format(i, numFiles))

def uploadFileCloud():
    from google.colab import files
    uploaded = files.upload()

Libaries

In [2]:
import requests
from lxml import html
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import cv2 as cv
import h5py
import os
import re
In [1]:
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential, load_model
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Concatenate
from keras.applications.vgg16 import VGG16
Using TensorFlow backend.

Data Preproccessing

Retrieval

In [3]:
PATH = "/Users/desiredewaele/Google Drive/Databases/Pokemon/" 
In [6]:
data = pd.read_csv(PATH+"list.csv", sep="\t")
pokemons = data.Ndex.reset_index(drop=True)
pokemons.tail()
Out[6]:
146      Dratini
147    Dragonair
148    Dragonite
149       Mewtwo
150          Mew
Name: Ndex, dtype: object
In [8]:
def getImages(items, path):
    print("Images retrieved for:")
    for item in items:
        pokepath = path+"images/"+item+"/"
        if not os.path.exists(pokepath): os.makedirs(pokepath)
        link = "https://www.bing.com/images/search?q=pokemon%20"+item+"&qs=n&form=QBIR"
        doc = requests.get(link)
        root = html.fromstring(doc.content)
        imgUrls = [x.get('href') for x in root.xpath('//a[@class="thumb"]')]
        imgUrls = [x for x in imgUrls if re.match("\.(png|jpg|jpeg)", x[-4:])]
        for i, imgUrl in enumerate(imgUrls):
            filename = re.sub(r'\W+', '', imgUrl)
            if not os.path.exists(pokepath+filename+".jpg"):
                try:
                    imgBytes = requests.get(imgUrl).content
                    imgArray = np.frombuffer(imgBytes, np.uint8)
                    image = cv.imdecode(imgArray, cv.IMREAD_COLOR)
                    image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
                    cv.imwrite(pokepath+filename+".jpg", image)
                except:
                    pass
        print(item, end=" ")
In [119]:
getImages(pokemons, PATH)
Images retrieved for:
Bulbasaur Ivysaur Venusaur Charmander Charmeleon Charizard Squirtle Wartortle Blastoise Caterpie Metapod Butterfree Weedle Kakuna Beedrill Pidgey Pidgeotto Pidgeot Rattata Raticate Spearow Fearow Ekans Arbok Pikachu Raichu Sandshrew Sandslash Nidoran♀ Nidorina Nidoqueen Nidoran♂ Nidorino Nidoking Clefairy Clefable Vulpix Ninetales Jigglypuff Wigglytuff Zubat Golbat Oddish Gloom Vileplume Paras Parasect Venonat Venomoth Diglett Dugtrio Meowth Persian Psyduck Golduck Mankey Primeape Growlithe Arcanine Poliwag Poliwhirl Poliwrath Abra Kadabra Alakazam Machop Machoke Machamp Bellsprout Weepinbell Victreebel Tentacool Tentacruel Geodude Graveler Golem Ponyta Rapidash Slowpoke Slowbro Magnemite Magneton Farfetch'd Doduo Dodrio Seel Dewgong Grimer Muk Shellder Cloyster Gastly Haunter Gengar Onix Drowzee Hypno Krabby Kingler Voltorb Electrode Exeggcute Exeggutor Cubone Marowak Hitmonlee Hitmonchan Lickitung Koffing Weezing Rhyhorn Rhydon Chansey Tangela Kangaskhan Horsea Seadra Goldeen Seaking Staryu Starmie Mr. Mime Scyther Jynx Electabuzz Magmar Pinsir Tauros Magikarp Gyarados Lapras Ditto Eevee Vaporeon Jolteon Flareon Porygon Omanyte Omastar Kabuto Kabutops Aerodactyl Snorlax Articuno Zapdos Moltres Dratini Dragonair Dragonite Mewtwo Mew 

Import

The whale dataset is proveded by Kaggle. It consists of a set of jpg-files with various resolutions, one for every image in the train and test set.

In [6]:
def importImages(folder, n=None, ratio=1.1634079267715058):
    images, labels = [], pd.Series()
    items = [f for f in os.listdir(folder) if f[0] != "."]
    for item in items:
        filenames = os.listdir(folder+item+"/")
        labels = labels.append(pd.Series(np.repeat(item, len(filenames))), ignore_index=True)
        for filename in filenames:
            image = cv.imread(folder+item+"/" + filename)
            y, x, _ = image.shape
            if x/y < ratio:
                pad = int((ratio*y - x)/2)
                image = np.pad(image, ((0,0), (pad, pad), (0,0)), 'edge')
            else:
                pad = int((1/ratio*x - y)/2)
                image = np.pad(image, ((pad, pad), (0,0), (0,0)), 'edge')
            image = cv.resize(image, (int(128*ratio), 128), interpolation=cv.INTER_LINEAR)
            image = np.divide(image, 255)
            image = np.expand_dims(image, 0)
            images.append(image)
            if n is not None and len(images) > n: break
    tensor = np.concatenate(images)
    return tensor, labels
In [7]:
PATH = "./pokemons/"
PATH = "/Users/desiredewaele/Google Drive/Databases/Pokemon/" 
tensor, labels = importImages(PATH + 'images/')
In [8]:
print("Shape of train data:", tensor.shape)
print("Shape of label data:", labels.shape)
Shape of train data: (4184, 128, 148, 3)
Shape of label data: (4184,)
In [9]:
def showImages(data, labels, loop=False, loops=1, grid=(3,7), figsize = (16,6)):
    n = len(data) / loops
    fig, ax = plt.subplots(grid[0], grid[1], figsize=figsize)
    for j in range(grid[1]):
        x = np.random.randint(n)
        for i in range(grid[0]):
            if loop: index = int(x+(i*n))
            else: index = np.random.randint(n)
            ax[i,j].imshow(data[index])
            ax[i,j].set_title("{}".format(labels[index]))
            ax[i,j].axis('off')
    plt.show()
In [10]:
showImages(tensor, labels)

Formatting

Some final steps are needed to feed them in a convolution network:

  • Adding a channel dimension for the images, even though our images are black and white. Keras needs this.
  • One-hot encode our labels to arrays of length 99.
  • Cut aside a validation set to optimalize our hyperparameters.
In [11]:
print("Shape of train data:", tensor.shape)
print("Shape of label data:", labels.shape)
Shape of train data: (4184, 128, 148, 3)
Shape of label data: (4184,)
In [12]:
dummies = pd.get_dummies(pd.DataFrame(labels))
target = dummies.values
targetLabels = np.array([x[2:] for x in dummies.columns.values])
In [13]:
print('Targets: {} '.format(target.shape))
Targets: (4184, 151) 
In [14]:
from sklearn.model_selection import train_test_split
tralidX, testX, tralidY, testY = train_test_split(tensor, target, test_size=151*5, stratify=target, random_state=1)
trainX, validX, trainY, validY = train_test_split(tralidX, tralidY, test_size=151*5, stratify=tralidY, random_state=1)
In [15]:
print('Training set:   {} {}'.format(trainX.shape, trainY.shape))
print('Validation set: {} {}'.format(validX.shape, validY.shape))
print('Testing set: {} {}'.format(testX.shape, testY.shape))
Training set:   (2674, 128, 148, 3) (2674, 151)
Validation set: (755, 128, 148, 3) (755, 151)
Testing set: (755, 128, 148, 3) (755, 151)

Augmentation

We now have 9850 images of size 128x289. By looping over the images, we can easily augment the data by:

  • Adding 15° rotations to the dataset.
  • Adding horizontal flips to the dataset.
  • Adding the grayscale version to the dataset.
In [16]:
BATCHSIZE = 32
In [17]:
trainAugmenter = ImageDataGenerator(
        rotation_range=10,
        width_shift_range=0.1,
        height_shift_range=0.1,
        shear_range=0.1,
        zoom_range=(1, 1.2),
        horizontal_flip=True)
validAugmenter = ImageDataGenerator()
testAugmenter = ImageDataGenerator()
In [18]:
trainGenerator = trainAugmenter.flow(trainX, trainY, BATCHSIZE)
validGenerator = validAugmenter.flow(validX, validY, BATCHSIZE)
testGenerator = testAugmenter.flow(testX, testY, BATCHSIZE)
In [ ]:
for batchX, batchY in trainGenerator:
    showImages(batchX, batchY)
    break

Convolutional Neural Network

After some iterations, these hyperparameters do the job.

In [19]:
modelPath = "/Users/desiredewaele/Google Drive/Models/" 
model = load_model(modelPath + 'pokeModel.h5')

Architecture

We choose an architecture with three onculutional layers. Each one is shrinked down by max-pooling while the depth increases. Next, we add two dense layers, accompanied by dropouts and activations.

In [128]:
model = Sequential()

# CONVOLUTION LAYERS
base = VGG16(include_top=False, weights='imagenet', input_shape=trainX.shape[1:])
for layer in base.layers:
    layer.trainable = False #Freeze
    model.add(layer)

# DENSE LAYERS
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(trainY.shape[1]*2, activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(trainY.shape[1], activation='softmax'))

model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
In [20]:
model.summary()
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_26 (InputLayer)        (None, 128, 148, 3)       0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 128, 148, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 128, 148, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 64, 74, 64)        0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 64, 74, 128)       73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 64, 74, 128)       147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 32, 37, 128)       0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 32, 37, 256)       295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 32, 37, 256)       590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 32, 37, 256)       590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 16, 18, 256)       0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 16, 18, 512)       1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 16, 18, 512)       2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 16, 18, 512)       2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 8, 9, 512)         0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 8, 9, 512)         2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 8, 9, 512)         2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 8, 9, 512)         2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 4, 4, 512)         0         
_________________________________________________________________
flatten_19 (Flatten)         (None, 8192)              0         
_________________________________________________________________
dropout_24 (Dropout)         (None, 8192)              0         
_________________________________________________________________
dense_30 (Dense)             (None, 298)               2441514   
_________________________________________________________________
dropout_25 (Dropout)         (None, 298)               0         
_________________________________________________________________
dense_31 (Dense)             (None, 149)               44551     
=================================================================
Total params: 17,200,753
Trainable params: 2,486,065
Non-trainable params: 14,714,688
_________________________________________________________________
In [38]:
NUMFILTERS = 16
FILTER = 3
In [43]:
model = Sequential([
    Conv2D(NUMFILTERS, (FILTER, FILTER), padding='same', activation='tanh', input_shape=(trainX.shape[1:])),
    MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),

    Conv2D(NUMFILTERS*2, (FILTER, FILTER), padding='same', activation='tanh'),
    MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),

    Conv2D(NUMFILTERS*4, (FILTER, FILTER), padding='same', activation='tanh'),
    MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),

    Conv2D(NUMFILTERS*8, (FILTER, FILTER), padding='same', activation='tanh'),
    MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),

    Flatten(),
    Dropout(0.5),
    Dense(trainY.shape[1]*2, activation='tanh'),
    Dropout(0.5),
    Dense(trainY.shape[1], activation='softmax'),
])

model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

This great Keras feature sumarizes our model.


Training

This model was trained on Colab.

In [130]:
EPOCHS = 30
In [153]:
history = model.fit_generator(trainGenerator, epochs=EPOCHS, validation_data=validGenerator)
Epoch 1/30
82/82 [==============================] - 264s 3s/step - loss: 2.2389 - acc: 0.4416 - val_loss: 2.2538 - val_acc: 0.5046
Epoch 2/30
82/82 [==============================] - 262s 3s/step - loss: 2.1018 - acc: 0.4731 - val_loss: 2.2269 - val_acc: 0.5232
Epoch 3/30
82/82 [==============================] - 260s 3s/step - loss: 2.0284 - acc: 0.4861 - val_loss: 2.2225 - val_acc: 0.5152
Epoch 4/30
82/82 [==============================] - 264s 3s/step - loss: 1.9903 - acc: 0.4958 - val_loss: 2.2124 - val_acc: 0.5258
Epoch 5/30
82/82 [==============================] - 250s 3s/step - loss: 1.9971 - acc: 0.4921 - val_loss: 2.1991 - val_acc: 0.5325
Epoch 6/30
82/82 [==============================] - 243s 3s/step - loss: 1.9606 - acc: 0.5044 - val_loss: 2.2216 - val_acc: 0.5245
Epoch 7/30
82/82 [==============================] - 246s 3s/step - loss: 1.9547 - acc: 0.5122 - val_loss: 2.2077 - val_acc: 0.5007
Epoch 8/30
82/82 [==============================] - 248s 3s/step - loss: 1.9644 - acc: 0.4996 - val_loss: 2.1829 - val_acc: 0.5126
Epoch 9/30
82/82 [==============================] - 250s 3s/step - loss: 1.9431 - acc: 0.5177 - val_loss: 2.1773 - val_acc: 0.5166
Epoch 10/30
82/82 [==============================] - 257s 3s/step - loss: 1.9229 - acc: 0.5168 - val_loss: 2.1943 - val_acc: 0.5298
Epoch 11/30
82/82 [==============================] - 258s 3s/step - loss: 1.8990 - acc: 0.5208 - val_loss: 2.2082 - val_acc: 0.5258
Epoch 12/30
82/82 [==============================] - 264s 3s/step - loss: 1.8621 - acc: 0.5280 - val_loss: 2.2150 - val_acc: 0.5086
Epoch 13/30
82/82 [==============================] - 248s 3s/step - loss: 1.8586 - acc: 0.5188 - val_loss: 2.1739 - val_acc: 0.5377
Epoch 14/30
82/82 [==============================] - 249s 3s/step - loss: 1.8723 - acc: 0.5208 - val_loss: 2.1562 - val_acc: 0.5391
Epoch 15/30
82/82 [==============================] - 249s 3s/step - loss: 1.8444 - acc: 0.5383 - val_loss: 2.1794 - val_acc: 0.5430
Epoch 16/30
82/82 [==============================] - 250s 3s/step - loss: 1.8221 - acc: 0.5358 - val_loss: 2.1682 - val_acc: 0.5325
Epoch 17/30
82/82 [==============================] - 249s 3s/step - loss: 1.7965 - acc: 0.5362 - val_loss: 2.1906 - val_acc: 0.5285
Epoch 18/30
82/82 [==============================] - 233s 3s/step - loss: 1.8335 - acc: 0.5351 - val_loss: 2.1840 - val_acc: 0.5245
Epoch 19/30
82/82 [==============================] - 245s 3s/step - loss: 1.8010 - acc: 0.5312 - val_loss: 2.1854 - val_acc: 0.5325
Epoch 20/30
82/82 [==============================] - 253s 3s/step - loss: 1.7302 - acc: 0.5484 - val_loss: 2.1898 - val_acc: 0.5179
Epoch 21/30
82/82 [==============================] - 255s 3s/step - loss: 1.8174 - acc: 0.5316 - val_loss: 2.1878 - val_acc: 0.5298
Epoch 22/30
82/82 [==============================] - 262s 3s/step - loss: 1.7641 - acc: 0.5319 - val_loss: 2.1862 - val_acc: 0.5391
Epoch 23/30
82/82 [==============================] - 247s 3s/step - loss: 1.7185 - acc: 0.5514 - val_loss: 2.1794 - val_acc: 0.5298
Epoch 24/30
82/82 [==============================] - 242s 3s/step - loss: 1.7293 - acc: 0.5497 - val_loss: 2.1818 - val_acc: 0.5351
Epoch 25/30
82/82 [==============================] - 243s 3s/step - loss: 1.6988 - acc: 0.5544 - val_loss: 2.1944 - val_acc: 0.5351
Epoch 26/30
82/82 [==============================] - 251s 3s/step - loss: 1.7400 - acc: 0.5551 - val_loss: 2.1729 - val_acc: 0.5325
Epoch 27/30
82/82 [==============================] - 256s 3s/step - loss: 1.7127 - acc: 0.5402 - val_loss: 2.1560 - val_acc: 0.5338
Epoch 28/30
82/82 [==============================] - 257s 3s/step - loss: 1.6577 - acc: 0.5603 - val_loss: 2.1420 - val_acc: 0.5391
Epoch 29/30
82/82 [==============================] - 253s 3s/step - loss: 1.6680 - acc: 0.5581 - val_loss: 2.1357 - val_acc: 0.5391
Epoch 30/30
82/82 [==============================] - 259s 3s/step - loss: 1.6671 - acc: 0.5749 - val_loss: 2.1875 - val_acc: 0.5232

Evaluation

Let's plot the accuracy and loss on both the training and validation set, during training time.

In [154]:
def showEvolution(history):
    plt.style.use('seaborn-white')

    fig = plt.figure(figsize=(17, 10)); ax1 = fig.add_subplot(221); ax2 = fig.add_subplot(222)
    positions = np.arange(1, EPOCHS+1)
    xticks = np.arange(0, EPOCHS+1, 2)
    ta, tl = history.history['acc'], history.history['loss']
    va, vl = history.history['val_acc'], history.history['val_loss']

    ax1.plot(positions, ta,".-", color='#2A6EA6', label="Training: {0:.2f}%".format(ta[-1]))
    ax1.plot(positions, va,".-", color='#FFA933', label="Validation: {0:.2f}%".format(va[-1]))
    ax1.grid(True); ax1.legend(); ax1.set_title("Accuracy per epoch");
    ax1.set_xticks(xticks)

    ax2.plot(positions, tl, ".-", color='#2A6EA6', label="Training: {0:.2f}".format(tl[-1]))
    ax2.plot(positions, vl,".-", color='#FFA933', label="Validation: {0:.2f}".format(vl[-1]))
    ax2.grid(True); ax2.legend(); ax2.set_title("Loss per epoch")
    ax2.set_xticks(xticks)

    plt.show()

First training iteration, batch size 8

In [133]:
showEvolution(history)

Second training iteration, batch size 32

In [155]:
showEvolution(history)

Our final metrics on both training and validation set are the following.

In [156]:
model.evaluate(trainX, trainY)
2618/2618 [==============================] - 202s 77ms/step
Out[156]:
[0.6046764576954182, 0.8571428572339261]
In [ ]:
model.evaluate(validX, validY)
In [158]:
model.evaluate(testX, testY)
755/755 [==============================] - 58s 76ms/step
Out[158]:
[2.255007185367559, 0.5059602650585554]

Download model to local drive.

In [159]:
modelPath = "./"#"/Users/desiredewaele/Google Drive/Models/" 
model.save(modelPath+'pokeModel.h5')
In [0]:
from google.colab import files
files.download('pokeModel.h5') 

Error Analysis

Lets use the model now to retrieve both the probabilities and the predicted classes on the validation set.

Image Level

In [160]:
probabilities = model.predict(validX)
evaluation = pd.DataFrame({"label":targetLabels[np.argmax(validY, axis=1)]})
for i in range(1,len(targetLabels)+1):evaluation["pred"+str(i)] = targetLabels[np.argsort(probabilities, axis=1)[:,-i]]
for i in range(1,len(targetLabels)+1):evaluation["prob"+str(i)] = np.sort(probabilities, axis=1)[:,-i]
evaluation.tail()
Out[160]:
label pred1 pred2 pred3 pred4 pred5 pred6 pred7 pred8 pred9 ... prob140 prob141 prob142 prob143 prob144 prob145 prob146 prob147 prob148 prob149
750 Marowak Spearow Abra Aerodactyl Alakazam Kadabra Kabutops Pinsir Golbat Nidoking ... 2.834051e-07 2.758886e-07 2.121331e-07 1.709338e-07 1.241167e-07 1.239379e-07 1.161628e-07 8.542465e-08 8.516988e-08 4.329619e-08
751 Ekans Arbok Ekans Poliwag Squirtle Gloom Tentacool Dragonair Machop Bellsprout ... 1.824887e-09 1.783212e-09 1.201894e-09 9.515126e-10 9.093245e-10 8.574798e-10 5.822685e-10 3.344899e-10 3.341787e-10 3.876243e-11
752 Vaporeon Pinsir Vaporeon Eevee Mew Meowth Pidgeotto Articuno Farfetchd Hypno ... 8.626441e-06 8.423877e-06 7.782755e-06 6.540768e-06 6.081324e-06 5.775869e-06 4.880995e-06 2.480239e-06 2.479305e-06 4.992541e-07
753 Kabutops Kabutops Aerodactyl Dodrio Mewtwo Scyther Electabuzz Pidgeot Mew Abra ... 1.853434e-08 1.604647e-08 1.150411e-08 1.146920e-08 7.865721e-09 3.812838e-09 3.559146e-09 2.395028e-09 7.993257e-10 4.972108e-10
754 Arcanine Machamp Rhydon Arcanine Primeape Rhyhorn Onix Growlithe Machoke Kangaskhan ... 1.142947e-07 6.241410e-08 6.180000e-08 4.906278e-08 3.890250e-08 2.959241e-08 1.430818e-08 9.848182e-09 6.757301e-09 5.917411e-09

5 rows × 299 columns

Class Level

In [161]:
def classAggregate(df):
    return pd.Series({"accuracy"+str(n): np.any(df.iloc[:,1:1+n].isin(df.label), axis=1).mean() for n in [1, 3, 5]})
In [162]:
evaluationClasses = evaluation.groupby('label').apply(classAggregate)
evaluationClasses.sort_values('accuracy5').head()
Out[162]:
accuracy1 accuracy3 accuracy5
label
Farfetchd 0.166667 0.166667 0.166667
Abra 0.000000 0.200000 0.200000
Weepinbell 0.000000 0.000000 0.200000
Magneton 0.000000 0.200000 0.200000
Ninetales 0.333333 0.333333 0.333333

Generic Level

In [163]:
classAggregate(evaluation)
Out[163]:
accuracy1    0.523179
accuracy3    0.694040
accuracy5    0.744371
dtype: float64

Now we can print some images with their true and predicted labels, as well as the probability distribution over the 99 classes.

In [164]:
def showPredictions(tensor, evaluation, mode='most certain', correct=True, label=None, predicted=None):
    if mode=="most certain": indeces = np.flip(np.argsort(evaluation.prob1), 0)
    elif mode=="most doubting": indeces = np.argsort(evaluation.prob1)
    if correct: indeces2 = np.where(evaluation.label == evaluation.pred1)[0]
    elif correct == False: indeces2 = np.where(evaluation.label != evaluation.pred1)[0]
    if correct is not None: indeces = indeces[np.in1d(indeces, indeces2)]
    if label is not None: indeces = indeces[np.in1d(indeces, np.where(evaluation.label==label)[0])]
    if predicted is not None: indeces = indeces[np.in1d(indeces, np.where(evaluation.pred1==predicted)[0])]
    
    nPlots = min(8, len(indeces))
    fig, axes = plt.subplots(2, nPlots, figsize = (nPlots*2, 4))
    for t in range(nPlots):
        if t >= len(indeces): break
        x = indeces.values[t]
        axes[0,t].imshow(tensor[x])
        axes[0,t].set_title("Truth: {}".format(evaluation.label[x]))
        axes[0,t].get_yaxis().set_visible(False); axes[0,t].get_xaxis().set_visible(False)
        axes[1,t].plot(evaluation.loc[x][152:152+151].values, lw=2)
        axes[1,t].set_title("Model: {}".format(evaluation.pred1[x]))
        axes[1,t].get_yaxis().set_visible(False); axes[1,t].set_ylim([0, 1]); axes[1,t].grid(True)
    plt.show()
In [165]:
showPredictions(validX, evaluation, mode='most certain', correct=True)
In [166]:
showPredictions(validX, evaluation, mode='most certain', correct=False)
In [167]:
showPredictions(validX, evaluation, mode='most doubting')