protoNN_torch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a52bf0d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (c) Microsoft Corporation. All rights reserved.\n",
    "# Licensed under the MIT license.\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import subprocess\n",
    "import glob\n",
    "import scipy.cluster\n",
    "import scipy.spatial\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn import metrics\n",
    "import argparse\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "df2dafad",
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiClassHingeLoss(logits, labels):\n",
    "    '''\n",
    "    MultiClassHingeLoss to match C++ Version - No pytorch internal version\n",
    "    '''\n",
    "    flatLogits = torch.reshape(logits, [-1, ])\n",
    "    labels_ = labels.argmax(dim=1)\n",
    "\n",
    "    correctId = torch.arange(labels.shape[0]).to(\n",
    "        logits.device) * labels.shape[1] + labels_\n",
    "    correctLogit = torch.gather(flatLogits, 0, correctId)\n",
    "\n",
    "    maxLabel = logits.argmax(dim=1)\n",
    "    top2, _ = torch.topk(logits, k=2, sorted=True)\n",
    "\n",
    "    wrongMaxLogit = torch.where((maxLabel == labels_), top2[:, 1], top2[:, 0])\n",
    "\n",
    "    return torch.mean(F.relu(1. + wrongMaxLogit - correctLogit))\n",
    "\n",
    "\n",
    "def crossEntropyLoss(logits, labels):\n",
    "    '''\n",
    "    Cross Entropy loss for MultiClass case in joint training for\n",
    "    faster convergence\n",
    "    '''\n",
    "    return F.cross_entropy(logits, labels.argmax(dim=1))\n",
    "\n",
    "\n",
    "def binaryHingeLoss(logits, labels):\n",
    "    '''\n",
    "    BinaryHingeLoss to match C++ Version - No pytorch internal version\n",
    "    '''\n",
    "    return torch.mean(F.relu(1.0 - (2 * labels - 1) * logits))\n",
    "\n",
    "\n",
    "def hardThreshold(A: torch.Tensor, s):\n",
    "    '''\n",
    "    Hard thresholds and modifies in-palce nn.Parameter A with sparsity s \n",
    "    '''\n",
    "    #PyTorch disallows numpy access/copy to tensors in graph.\n",
    "    #.detach() creates a new tensor not attached to the graph.\n",
    "    A_ = A.data.cpu().detach().numpy().ravel()    \n",
    "    if len(A_) > 0:\n",
    "        th = np.percentile(np.abs(A_), (1 - s) * 100.0, interpolation='higher')\n",
    "        A_[np.abs(A_) < th] = 0.0\n",
    "    A_ = A_.reshape(A.shape)\n",
    "    return torch.tensor(A_, requires_grad=True)\n",
    "\n",
    "def supportBasedThreshold(dst: torch.Tensor, src: torch.Tensor):\n",
    "    '''\n",
    "    zero out entries in dst.data that are zeros in src tensor\n",
    "    '''\n",
    "    return copySupport(src, dst.data)\n",
    "\n",
    "def copySupport(src, dst):\n",
    "    '''\n",
    "    zero out entries in dst.data that are zeros in src tensor\n",
    "    '''\n",
    "    zeroSupport = (src.view(-1) == 0.0).nonzero()\n",
    "    dst = dst.reshape(-1)\n",
    "    dst[zeroSupport] = 0\n",
    "    dst = dst.reshape(src.shape)\n",
    "    del zeroSupport\n",
    "    return dst\n",
    "\n",
    "\n",
    "def estimateNNZ(A, s, bytesPerVar=4):\n",
    "    '''\n",
    "    Returns # of non-zeros and representative size of the tensor\n",
    "    Uses dense for s >= 0.5 - 4 byte\n",
    "    Else uses sparse - 8 byte\n",
    "    '''\n",
    "    params = 1\n",
    "    hasSparse = False\n",
    "    for i in range(0, len(A.shape)):\n",
    "        params *= int(A.shape[i])\n",
    "    if s < 0.5:\n",
    "        nnZ = np.ceil(params * s)\n",
    "        hasSparse = True\n",
    "        return nnZ, nnZ * 2 * bytesPerVar, hasSparse\n",
    "    else:\n",
    "        nnZ = params\n",
    "        return nnZ, nnZ * bytesPerVar, hasSparse\n",
    "\n",
    "\n",
    "def countNNZ(A: torch.nn.Parameter, isSparse):\n",
    "    '''\n",
    "    Returns # of non-zeros \n",
    "    '''\n",
    "    A_ = A.detach().numpy()\n",
    "    if isSparse:\n",
    "        return np.count_nonzero(A_)\n",
    "    else:\n",
    "        nnzs = 1\n",
    "        for i in range(0, len(A.shape)):\n",
    "            nnzs *= int(A.shape[i])\n",
    "        return nnzs\n",
    "\n",
    "def restructreMatrixBonsaiSeeDot(A, nClasses, nNodes):\n",
    "    '''\n",
    "    Restructures a matrix from [nNodes*nClasses, Proj] to\n",
    "    [nClasses*nNodes, Proj] for SeeDot\n",
    "    '''\n",
    "    tempMatrix = np.zeros(A.shape)\n",
    "    rowIndex = 0\n",
    "\n",
    "    for i in range(0, nClasses):\n",
    "        for j in range(0, nNodes):\n",
    "            tempMatrix[rowIndex] = A[j * nClasses + i]\n",
    "            rowIndex += 1\n",
    "\n",
    "    return tempMatrix\n",
    "\n",
    "class TriangularLR(optim.lr_scheduler._LRScheduler):\n",
    "    def __init__(self, optimizer, stepsize, lr_min, lr_max, gamma):\n",
    "        self.stepsize = stepsize\n",
    "        self.lr_min = lr_min\n",
    "        self.lr_max = lr_max\n",
    "        self.gamma = gamma\n",
    "        super(TriangularLR, self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        it = self.last_epoch\n",
    "        cycle = math.floor(1 + it / (2 * self.stepsize))\n",
    "        x = abs(it / self.stepsize - 2 * cycle + 1)\n",
    "        decayed_range = (self.lr_max - self.lr_min) * self.gamma ** (it / 3)\n",
    "        lr = self.lr_min + decayed_range * x\n",
    "        return [lr]\n",
    "\n",
    "class ExponentialResettingLR(optim.lr_scheduler._LRScheduler):\n",
    "    def __init__(self, optimizer, gamma, reset_epoch):\n",
    "        self.gamma = gamma\n",
    "        self.reset_epoch = int(reset_epoch)\n",
    "        super(ExponentialResettingLR, self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        epoch = self.last_epoch\n",
    "        if epoch > self.reset_epoch:\n",
    "            epoch -= self.reset_epoch\n",
    "        return [base_lr * self.gamma ** epoch\n",
    "                for base_lr in self.base_lrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f94d4cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProtoNN(nn.Module):\n",
    "    def __init__(self, inputDimension, projectionDimension, numPrototypes,\n",
    "                 numOutputLabels, gamma, W=None, B=None, Z=None):\n",
    "        '''\n",
    "        Forward computation graph for ProtoNN.\n",
    "        inputDimension: Input data dimension or feature dimension.\n",
    "        projectionDimension: hyperparameter\n",
    "        numPrototypes: hyperparameter\n",
    "        numOutputLabels: The number of output labels or classes\n",
    "        W, B, Z: Numpy matrices that can be used to initialize\n",
    "            projection matrix(W), prototype matrix (B) and prototype labels\n",
    "            matrix (B).\n",
    "            Expected Dimensions:\n",
    "                W   inputDimension (d) x projectionDimension (d_cap)\n",
    "                B   projectionDimension (d_cap) x numPrototypes (m)\n",
    "                Z   numOutputLabels (L) x numPrototypes (m)\n",
    "        '''\n",
    "        super(ProtoNN, self).__init__()\n",
    "        self.__d = inputDimension\n",
    "        self.__d_cap = projectionDimension\n",
    "        self.__m = numPrototypes\n",
    "        self.__L = numOutputLabels\n",
    "\n",
    "        self.W, self.B, self.Z = None, None, None\n",
    "        self.gamma = gamma\n",
    "\n",
    "        self.__validInit = False\n",
    "        self.__initWBZ(W, B, Z)\n",
    "        self.__validateInit()\n",
    "\n",
    "    def __validateInit(self):\n",
    "        self.__validinit = False\n",
    "        errmsg = \"Dimensions mismatch! Should be W[d, d_cap]\"\n",
    "        errmsg+= \", B[d_cap, m] and Z[L, m]\"\n",
    "        d, d_cap, m, L, _ = self.getHyperParams()\n",
    "        assert self.W.shape[0] == d, errmsg\n",
    "        assert self.W.shape[1] == d_cap, errmsg\n",
    "        assert self.B.shape[0] == d_cap, errmsg\n",
    "        assert self.B.shape[1] == m, errmsg\n",
    "        assert self.Z.shape[0] == L, errmsg\n",
    "        assert self.Z.shape[1] == m, errmsg\n",
    "        self.__validInit = True\n",
    "\n",
    "    def __initWBZ(self, inW, inB, inZ):\n",
    "        if inW is None:\n",
    "            self.W = torch.randn([self.__d, self.__d_cap])\n",
    "            self.W = nn.Parameter(self.W)\n",
    "        else:\n",
    "            self.W = nn.Parameter(torch.from_numpy(inW.astype(np.float32)))\n",
    "\n",
    "        if inB is None:\n",
    "            self.B = torch.randn([self.__d_cap, self.__m])\n",
    "            self.B = nn.Parameter(self.B)\n",
    "        else:\n",
    "            self.B = nn.Parameter(torch.from_numpy(inB.astype(np.float32)))\n",
    "\n",
    "        if inZ is None:\n",
    "            self.Z = torch.randn([self.__L, self.__m])\n",
    "            self.Z = nn.Parameter(self.Z)\n",
    "        else:\n",
    "            self.Z = nn.Parameter(torch.from_numpy(inZ.astype(np.float32)))\n",
    "\n",
    "    def getHyperParams(self):\n",
    "        '''\n",
    "        Returns the model hyperparameters:\n",
    "            [inputDimension, projectionDimension, numPrototypes,\n",
    "            numOutputLabels, gamma]\n",
    "        '''\n",
    "        d =  self.__d\n",
    "        dcap = self.__d_cap\n",
    "        m = self.__m\n",
    "        L = self.__L\n",
    "        return d, dcap, m, L, self.gamma\n",
    "\n",
    "    def getModelMatrices(self):\n",
    "        '''\n",
    "        Returns model matrices, which can then be evaluated to obtain\n",
    "        corresponding numpy arrays.  These can then be exported as part of\n",
    "        other implementations of ProtonNN, for instance a C++ implementation or\n",
    "        pure python implementation.\n",
    "        Returns\n",
    "            [ProjectionMatrix (W), prototypeMatrix (B),\n",
    "             prototypeLabelsMatrix (Z), gamma]\n",
    "        '''\n",
    "        return self.W, self.B, self.Z, self.gamma\n",
    "\n",
    "    def forward(self, X):\n",
    "        '''\n",
    "        This method is responsible for construction of the forward computation\n",
    "        graph. The end point of the computation graph, or in other words the\n",
    "        output operator for the forward computation is returned.\n",
    "        X: Input of shape [-1, inputDimension]\n",
    "        returns: The forward computation outputs, self.protoNNOut\n",
    "        '''\n",
    "        assert self.__validInit is True, \"Initialization failed!\"\n",
    "\n",
    "        W, B, Z, gamma = self.W, self.B, self.Z, self.gamma\n",
    "        WX = torch.matmul(X, W)\n",
    "        dim = [-1, WX.shape[1], 1]\n",
    "        WX = torch.reshape(WX, dim)\n",
    "        dim = [1, B.shape[0], -1]\n",
    "        B_ = torch.reshape(B, dim)\n",
    "        l2sim = B_ - WX\n",
    "        l2sim = torch.pow(l2sim, 2)\n",
    "        l2sim = torch.sum(l2sim, dim=1, keepdim=True)\n",
    "        self.l2sim = l2sim\n",
    "        gammal2sim = (-1 * gamma * gamma) * l2sim\n",
    "        M = torch.exp(gammal2sim)\n",
    "        dim = [1] + list(Z.shape)\n",
    "        Z_ = torch.reshape(Z, dim)\n",
    "        y = Z_ * M\n",
    "        y = torch.sum(y, dim=2)\n",
    "        return y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8411cb15",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProtoNNTrainer:\n",
    "\n",
    "    def __init__(self, protoNNObj, regW, regB, regZ, sparcityW, sparcityB,\n",
    "                 sparcityZ, learningRate, lossType='l2', device=None):\n",
    "        '''\n",
    "        A wrapper for the various techniques used for training ProtoNN. This\n",
    "        subsumes both the responsibility of loss graph construction and\n",
    "        performing training. The original training routine that is part of the\n",
    "        C++ implementation of EdgeML used iterative hard thresholding (IHT),\n",
    "        gamma estimation through median heuristic and other tricks for\n",
    "        training ProtoNN. This module implements the same in pytorch\n",
    "        and python.\n",
    "        protoNNObj: An instance of ProtoNN class defining the forward\n",
    "            computation graph. The loss functions and training routines will be\n",
    "            attached to this instance.\n",
    "        regW, regB, regZ: Regularization constants for W, B, and\n",
    "            Z matrices of protoNN.\n",
    "        sparcityW, sparcityB, sparcityZ: Sparsity constraints\n",
    "            for W, B and Z matrices. A value between 0 (exclusive) and 1\n",
    "            (inclusive) is expected. A value of 1 indicates dense training.\n",
    "        learningRate: Initial learning rate for ADAM optimizer.\n",
    "        X, Y : Placeholders for data and labels.\n",
    "            X [-1, featureDimension]\n",
    "            Y [-1, num Labels]\n",
    "        lossType: ['l2', 'xentropy']\n",
    "        '''\n",
    "        self.protoNNObj = protoNNObj\n",
    "        self.__regW = regW\n",
    "        self.__regB = regB\n",
    "        self.__regZ = regZ\n",
    "        self.__sW = sparcityW\n",
    "        self.__sB = sparcityB\n",
    "        self.__sZ = sparcityZ\n",
    "        self.__lR = learningRate\n",
    "        self.sparseTraining = True\n",
    "        if (sparcityW == 1.0) and (sparcityB == 1.0) and (sparcityZ == 1.0):\n",
    "            self.sparseTraining = False\n",
    "            print(\"Sparse training disabled.\", file=sys.stderr)\n",
    "        self.W_th = None\n",
    "        self.B_th = None\n",
    "        self.Z_th = None\n",
    "        self.__lossType = lossType\n",
    "        self.optimizer = self.__optimizer()\n",
    "        self.lossCriterion = None\n",
    "        #assert lossType in ['l2', 'xentropy']\n",
    "        if lossType == 'l2':\n",
    "            self.lossCriterion = torch.nn.MSELoss()\n",
    "            print(\"Using L2 (MSE) loss\")\n",
    "        else :\n",
    "            self.lossCriterion = torch.nn.CrossEntropyLoss()\n",
    "            print(\"Using x-entropy loss\")\n",
    "        self.__validInit = False\n",
    "        self.__validInit = self.__validateInit()\n",
    "        if device is None:\n",
    "            self.device = \"cpu\"\n",
    "        else:\n",
    "            self.device = device\n",
    "\n",
    "    def __validateInit(self):\n",
    "        assert self.__validInit == False\n",
    "        msg = \"Sparsity values should be between 0 and 1 (both inclusive)\"\n",
    "        assert 0 <= self.__sW <= 1, msg\n",
    "        assert 0 <= self.__sB <= 1, msg\n",
    "        assert 0 <= self.__sZ <= 1, msg\n",
    "        return True\n",
    "\n",
    "    def __optimizer(self):\n",
    "        optimizer = torch.optim.Adam(self.protoNNObj.parameters(),\n",
    "                                     lr=self.__lR)\n",
    "        return optimizer\n",
    "\n",
    "    def loss(self, logits, labels_or_target):\n",
    "        labels = labels_or_target\n",
    "        assert len(logits) == len(labels)\n",
    "        assert len(labels.shape) == 2\n",
    "        assert len(logits.shape) == 2\n",
    "        regLoss = (self.__regW * (torch.norm(self.protoNNObj.W)**2) +\n",
    "                   self.__regB * (torch.norm(self.protoNNObj.B)**2) +\n",
    "                   self.__regZ * (torch.norm(self.protoNNObj.Z)**2))\n",
    "        if self.__lossType == 'xentropy':\n",
    "            _, labels = torch.max(labels, dim=1)\n",
    "            assert len(labels.shape)== 1\n",
    "        loss = self.lossCriterion(logits, labels) + regLoss\n",
    "        return loss\n",
    "\n",
    "    def accuracy(self, predictions, labels):\n",
    "        '''\n",
    "        Returns accuracy and number of correct predictions.\n",
    "        '''\n",
    "        assert len(predictions.shape) == 1\n",
    "        assert len(labels.shape) == 1\n",
    "        assert len(predictions) == len(labels)\n",
    "        correct = (predictions == labels).double()\n",
    "        numCorrect = torch.sum(correct)\n",
    "        acc = torch.mean(correct)\n",
    "        return acc, numCorrect\n",
    "\n",
    "    def hardThreshold(self):\n",
    "        prtn = self.protoNNObj\n",
    "        W, B, Z = prtn.W.data, prtn.B.data, prtn.Z.data\n",
    "        newW = hardThreshold(W, self.__sW)\n",
    "        newB = hardThreshold(B, self.__sB)\n",
    "        newZ = hardThreshold(Z, self.__sZ)\n",
    "        prtn.W.data = torch.FloatTensor(newW).to(self.device)\n",
    "        prtn.B.data = torch.FloatTensor(newB).to(self.device)\n",
    "        prtn.Z.data = torch.FloatTensor(newZ).to(self.device)\n",
    "        \n",
    "    def estimate_confusion_matrix(decision,y_val):\n",
    "        L=3; D=3\n",
    "        ConfusionMatrix=np.zeros((D,L))\n",
    "        for d in range(D):\n",
    "            for l in range(L):\n",
    "                idx=((decision==d) & (y_val == l)) \n",
    "\n",
    "                if y_val[l]==0:\n",
    "                    ConfusionMatrix[d][l]=sum(idx)\n",
    "                elif y_val[l]==1:\n",
    "                    ConfusionMatrix[d][l]=sum(idx)\n",
    "                else:\n",
    "                    ConfusionMatrix[d][l]= sum(idx)# / sum((labels == l))\n",
    "        return ConfusionMatrix\n",
    "\n",
    "    def train(self, batchSize, epochs, x_train, x_val, y_train, y_val,\n",
    "              printStep=10, valStep=1):\n",
    "        '''\n",
    "        Performs dense training of ProtoNN followed by iterative hard\n",
    "        thresholding to enforce sparsity constraints.\n",
    "        batchSize: Batch size per update\n",
    "        epochs : The number of epochs to run training for. One epoch is\n",
    "            defined as one pass over the entire training data.\n",
    "        x_train, x_val, y_train, y_val: The numpy array containing train and\n",
    "            validation data. x data is assumed to in of shape [-1,\n",
    "            featureDimension] while y should have shape [-1, numberLabels].\n",
    "        printStep: Number of batches between echoing of loss and train accuracy.\n",
    "        valStep: Number of epochs between evaluations on validation set.\n",
    "        '''\n",
    "        d, dcap, m, L, _ = self.protoNNObj.getHyperParams()\n",
    "        assert batchSize >= 1, 'Batch size should be positive integer'\n",
    "        assert epochs >= 1, 'Total epochs should be positive integer'\n",
    "        assert x_train.ndim == 2, 'Expected training data to be of rank 2'\n",
    "        assert x_train.shape[1] == d, 'Expected x_train to be [-1, %d]' % d\n",
    "        assert x_val.ndim == 2, 'Expected validation data to be of rank 2'\n",
    "        assert x_val.shape[1] == d, 'Expected x_val to be [-1, %d]' % d\n",
    "        assert y_train.ndim == 2, 'Expected training labels to be of rank 2'\n",
    "        assert y_train.shape[1] == L, 'Expected y_train to be [-1, %d]' % L\n",
    "        assert y_val.ndim == 2, 'Expected validation labels to be of rank 2'\n",
    "        assert y_val.shape[1] == L, 'Expected y_val to be [-1, %d]' % L\n",
    "\n",
    "        trainNumBatches = int(np.ceil(len(x_train) / batchSize))\n",
    "        valNumBatches = int(np.ceil(len(x_val) / batchSize))\n",
    "        x_train_batches = np.array_split(x_train, trainNumBatches)\n",
    "        y_train_batches = np.array_split(y_train, trainNumBatches)\n",
    "        x_val_batches = np.array_split(x_val, valNumBatches)\n",
    "        y_val_batches = np.array_split(y_val, valNumBatches)\n",
    "        vacc=0; pred=0\n",
    "        for epoch in range(epochs):\n",
    "            for i in range(len(x_train_batches)):\n",
    "                x_batch, y_batch = x_train_batches[i], y_train_batches[i]\n",
    "                x_batch, y_batch = torch.Tensor(x_batch), torch.Tensor(y_batch)\n",
    "                x_batch, y_batch = x_batch.to(self.device), y_batch.to(self.device)\n",
    "                self.optimizer.zero_grad()\n",
    "                logits = self.protoNNObj.forward(x_batch)\n",
    "                loss = self.loss(logits, y_batch)\n",
    "                loss.backward()\n",
    "                self.optimizer.step()\n",
    "                _, predictions = torch.max(logits, dim=1)\n",
    "                _, target = torch.max(y_batch, dim=1)\n",
    "                acc, _ = self.accuracy(predictions, target)\n",
    "                if i % printStep == 0:\n",
    "                    print(\"Epoch %d batch %d loss %f acc %f\" % (epoch, i, loss,acc))\n",
    "            # Perform IHT Here.\n",
    "            if self.sparseTraining:\n",
    "                self.hardThreshold()\n",
    "            # Perform validation set evaluation\n",
    "            if (epoch + 1) % valStep == 0:\n",
    "                numCorrect = 0\n",
    "                for i in range(len(x_val_batches)):\n",
    "                    x_batch, y_batch = x_val_batches[i], y_val_batches[i]\n",
    "                    x_batch, y_batch = torch.Tensor(x_batch), torch.Tensor(y_batch)\n",
    "                    x_batch, y_batch = x_batch.to(self.device), y_batch.to(self.device)\n",
    "                    logits = self.protoNNObj.forward(x_batch)\n",
    "                    _, predictions = torch.max(logits, dim=1)\n",
    "                    _, target = torch.max(y_batch, dim=1)\n",
    "                    _, count = self.accuracy(predictions, target)\n",
    "                    numCorrect += count\n",
    "                    vacc +=  (numCorrect / len(x_val))\n",
    "#                     pred+=predictions\n",
    "                print(\"Validation accuracy: %f\" % (numCorrect / len(x_val)))\n",
    "#         print(len(pred), (y_val).shape)\n",
    "\n",
    "        print(\"Accuracy: \", vacc/ len(y_val_batches))\n",
    "        print(\"Loss: \", loss/ len(y_val_batches))\n",
    "#         estimate_confusion_matrix(y_val, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6ebfdf70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Dimension:  45\n",
      "Num classes:  3\n"
     ]
    }
   ],
   "source": [
    "#Extract the features and the predictors\n",
    "time_data = pd.read_csv(\"/Users/vanshika/Downloads/dataset_fog_release/dataset_fog_release/dataset/time.csv\")\n",
    "target = time_data['0'] #action 0, 1, 2\n",
    "time_data = time_data.drop(['0'], axis = 1)\n",
    "\n",
    "scaler = MinMaxScaler((-1, 1)) #scaling\n",
    "X = scaler.fit_transform(time_data)\n",
    "Y = target\n",
    "\n",
    "#Split training data \n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = .25, random_state = 7)\n",
    "\n",
    "dataDimension = X.shape[1]\n",
    "numClasses = len(np.unique(Y))\n",
    "\n",
    "print(\"Feature Dimension: \", dataDimension)\n",
    "print(\"Num classes: \", numClasses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "216b865d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(target)):\n",
    "#     if target[i] == 0:\n",
    "#         target= target.drop(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a60015c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# one hot y-train\n",
    "lab = Y_train.astype('uint8')\n",
    "lab = np.array(lab) - min(lab)\n",
    "lab_ = np.zeros((X_train.shape[0], numClasses))\n",
    "lab_[np.arange(X_train.shape[0]), lab] = 1\n",
    "y_train = lab_\n",
    "\n",
    "# one hot y-test\n",
    "lab = Y_test.astype('uint8')\n",
    "lab = np.array(lab) - min(lab)\n",
    "lab_ = np.zeros((X_test.shape[0], numClasses))\n",
    "lab_[np.arange(X_test.shape[0]), lab] = 1\n",
    "y_test = lab_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "453dafcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECTION_DIM = 5 #d^\n",
    "NUM_PROTOTYPES = 40 #m\n",
    "REG_W = 0.000005\n",
    "REG_B = 0.0\n",
    "REG_Z = 0.00005\n",
    "SPAR_W = 1.0\n",
    "SPAR_B = 0.8\n",
    "SPAR_Z = 0.8\n",
    "LEARNING_RATE = 0.05\n",
    "NUM_EPOCHS = 200\n",
    "BATCH_SIZE = 32\n",
    "GAMMA = 0.024634"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cf41ecbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#utils\n",
    "\n",
    "def medianHeuristic(data, projectionDimension, numPrototypes, W_init=None):\n",
    "    '''\n",
    "    This method can be used to estimate gamma for ProtoNN. An approximation to\n",
    "    median heuristic is used here.\n",
    "    1. First the data is collapsed into the projectionDimension by W_init. If\n",
    "    W_init is not provided, it is initialized from a random normal(0, 1). Hence\n",
    "    data normalization is essential.\n",
    "    2. Prototype are computed by running a  k-means clustering on the projected\n",
    "    data.\n",
    "    3. The median distance is then estimated by calculating median distance\n",
    "    between prototypes and projected data points.\n",
    "\n",
    "    data needs to be [-1, numFeats]\n",
    "    If using this method to initialize gamma, please use the W and B as well.\n",
    "\n",
    "    TODO: Return estimate of Z (prototype labels) based on cluster centroids\n",
    "    andand labels\n",
    "\n",
    "    TODO: Clustering fails due to singularity error if projecting upwards\n",
    "\n",
    "    W [dxd_cap]\n",
    "    B [d_cap, m]\n",
    "    returns gamma, W, B\n",
    "    '''\n",
    "    assert data.ndim == 2\n",
    "    X = data\n",
    "    featDim = data.shape[1]\n",
    "    if projectionDimension > featDim:\n",
    "        print(\"Warning: Projection dimension > feature dimension. Gamma\")\n",
    "        print(\"\\t estimation due to median heuristic could fail.\")\n",
    "        print(\"\\tTo retain the projection dataDimension, provide\")\n",
    "        print(\"\\ta value for gamma.\")\n",
    "\n",
    "    if W_init is None:\n",
    "        W_init = np.random.normal(size=[featDim, projectionDimension])\n",
    "    W = W_init\n",
    "    XW = np.matmul(X, W)\n",
    "    assert XW.shape[1] == projectionDimension\n",
    "    assert XW.shape[0] == len(X)\n",
    "    # Requires [N x d_cap] data matrix of N observations of d_cap-dimension and\n",
    "    # the number of centroids m. Returns, [n x d_cap] centroids and\n",
    "    # elementwise center information.\n",
    "    B, centers = scipy.cluster.vq.kmeans2(XW, numPrototypes)\n",
    "    # Requires two matrices. Number of observations x dimension of observation\n",
    "    # space. Distances[i,j] is the distance between XW[i] and B[j]\n",
    "    distances = scipy.spatial.distance.cdist(XW, B, metric='euclidean')\n",
    "    distances = np.reshape(distances, [-1])\n",
    "    gamma = np.median(distances)\n",
    "    gamma = 1 / (2.5 * gamma)\n",
    "    return gamma.astype('float32'), W.astype('float32'), B.T.astype('float32')\n",
    "\n",
    "#helper methods\n",
    "def getGamma(gammaInit, projectionDim, dataDim, numPrototypes, x_train):\n",
    "    if gammaInit is None:\n",
    "        print(\"Using median heuristic to estimate gamma.\")\n",
    "        gamma, W, B = medianHeuristic(x_train, projectionDim,\n",
    "                                            numPrototypes)\n",
    "        print(\"Gamma estimate is: %f\" % gamma)\n",
    "        return W, B, gamma\n",
    "    return None, None, gammaInit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "27293cb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "W, B, gamma = getGamma(GAMMA, PROJECTION_DIM, dataDimension,\n",
    "                       NUM_PROTOTYPES, X_train) #x_train for small gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "869c383c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using L2 (MSE) loss\n",
      "Epoch 0 batch 0 loss 13.868752 acc 0.031250\n",
      "Epoch 0 batch 100 loss 0.133627 acc 0.843750\n",
      "Epoch 0 batch 200 loss 0.117879 acc 0.812500\n",
      "Epoch 0 batch 300 loss 0.087056 acc 0.875000\n",
      "Epoch 0 batch 400 loss 0.123929 acc 0.781250\n",
      "Epoch 0 batch 500 loss 0.067556 acc 0.906250\n",
      "Epoch 0 batch 600 loss 0.080463 acc 0.875000\n",
      "Epoch 0 batch 700 loss 0.095900 acc 0.843750\n",
      "Epoch 0 batch 800 loss 0.080475 acc 0.875000\n",
      "Epoch 0 batch 900 loss 0.083544 acc 0.875000\n",
      "Epoch 0 batch 1000 loss 0.093888 acc 0.843750\n",
      "Epoch 0 batch 1100 loss 0.118266 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 1 batch 0 loss 0.053911 acc 0.937500\n",
      "Epoch 1 batch 100 loss 0.095737 acc 0.843750\n",
      "Epoch 1 batch 200 loss 0.111083 acc 0.812500\n",
      "Epoch 1 batch 300 loss 0.081878 acc 0.875000\n",
      "Epoch 1 batch 400 loss 0.115514 acc 0.781250\n",
      "Epoch 1 batch 500 loss 0.076038 acc 0.906250\n",
      "Epoch 1 batch 600 loss 0.087231 acc 0.875000\n",
      "Epoch 1 batch 700 loss 0.095529 acc 0.843750\n",
      "Epoch 1 batch 800 loss 0.078349 acc 0.875000\n",
      "Epoch 1 batch 900 loss 0.080629 acc 0.875000\n",
      "Epoch 1 batch 1000 loss 0.091914 acc 0.843750\n",
      "Epoch 1 batch 1100 loss 0.117247 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 2 batch 0 loss 0.049188 acc 0.937500\n",
      "Epoch 2 batch 100 loss 0.091529 acc 0.843750\n",
      "Epoch 2 batch 200 loss 0.108719 acc 0.812500\n",
      "Epoch 2 batch 300 loss 0.082512 acc 0.875000\n",
      "Epoch 2 batch 400 loss 0.115316 acc 0.781250\n",
      "Epoch 2 batch 500 loss 0.078957 acc 0.906250\n",
      "Epoch 2 batch 600 loss 0.087510 acc 0.875000\n",
      "Epoch 2 batch 700 loss 0.093151 acc 0.843750\n",
      "Epoch 2 batch 800 loss 0.078155 acc 0.875000\n",
      "Epoch 2 batch 900 loss 0.081615 acc 0.875000\n",
      "Epoch 2 batch 1000 loss 0.089488 acc 0.843750\n",
      "Epoch 2 batch 1100 loss 0.115070 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 3 batch 0 loss 0.053997 acc 0.937500\n",
      "Epoch 3 batch 100 loss 0.089164 acc 0.843750\n",
      "Epoch 3 batch 200 loss 0.106166 acc 0.812500\n",
      "Epoch 3 batch 300 loss 0.078936 acc 0.875000\n",
      "Epoch 3 batch 400 loss 0.115240 acc 0.781250\n",
      "Epoch 3 batch 500 loss 0.076441 acc 0.906250\n",
      "Epoch 3 batch 600 loss 0.089174 acc 0.875000\n",
      "Epoch 3 batch 700 loss 0.092327 acc 0.843750\n",
      "Epoch 3 batch 800 loss 0.080414 acc 0.875000\n",
      "Epoch 3 batch 900 loss 0.079700 acc 0.875000\n",
      "Epoch 3 batch 1000 loss 0.087393 acc 0.843750\n",
      "Epoch 3 batch 1100 loss 0.113704 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 4 batch 0 loss 0.059677 acc 0.937500\n",
      "Epoch 4 batch 100 loss 0.087921 acc 0.843750\n",
      "Epoch 4 batch 200 loss 0.104203 acc 0.812500\n",
      "Epoch 4 batch 300 loss 0.078266 acc 0.875000\n",
      "Epoch 4 batch 400 loss 0.114302 acc 0.781250\n",
      "Epoch 4 batch 500 loss 0.073922 acc 0.906250\n",
      "Epoch 4 batch 600 loss 0.088996 acc 0.875000\n",
      "Epoch 4 batch 700 loss 0.092036 acc 0.843750\n",
      "Epoch 4 batch 800 loss 0.081887 acc 0.875000\n",
      "Epoch 4 batch 900 loss 0.076939 acc 0.875000\n",
      "Epoch 4 batch 1000 loss 0.084683 acc 0.843750\n",
      "Epoch 4 batch 1100 loss 0.110965 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 5 batch 0 loss 0.060362 acc 0.937500\n",
      "Epoch 5 batch 100 loss 0.087255 acc 0.843750\n",
      "Epoch 5 batch 200 loss 0.104192 acc 0.812500\n",
      "Epoch 5 batch 300 loss 0.077686 acc 0.875000\n",
      "Epoch 5 batch 400 loss 0.112786 acc 0.781250\n",
      "Epoch 5 batch 500 loss 0.072721 acc 0.906250\n",
      "Epoch 5 batch 600 loss 0.090241 acc 0.875000\n",
      "Epoch 5 batch 700 loss 0.090628 acc 0.843750\n",
      "Epoch 5 batch 800 loss 0.083820 acc 0.875000\n",
      "Epoch 5 batch 900 loss 0.076818 acc 0.875000\n",
      "Epoch 5 batch 1000 loss 0.082918 acc 0.843750\n",
      "Epoch 5 batch 1100 loss 0.108573 acc 0.781250\n",
      "Validation accuracy: 0.825322\n",
      "Epoch 6 batch 0 loss 0.064744 acc 0.937500\n",
      "Epoch 6 batch 100 loss 0.086244 acc 0.843750\n",
      "Epoch 6 batch 200 loss 0.104648 acc 0.812500\n",
      "Epoch 6 batch 300 loss 0.076285 acc 0.875000\n",
      "Epoch 6 batch 400 loss 0.111855 acc 0.781250\n",
      "Epoch 6 batch 500 loss 0.071497 acc 0.906250\n",
      "Epoch 6 batch 600 loss 0.091813 acc 0.875000\n",
      "Epoch 6 batch 700 loss 0.089954 acc 0.843750\n",
      "Epoch 6 batch 800 loss 0.087248 acc 0.875000\n",
      "Epoch 6 batch 900 loss 0.075388 acc 0.875000\n",
      "Epoch 6 batch 1000 loss 0.081824 acc 0.843750\n",
      "Epoch 6 batch 1100 loss 0.106214 acc 0.781250\n",
      "Validation accuracy: 0.827769\n",
      "Epoch 7 batch 0 loss 0.065386 acc 0.937500\n",
      "Epoch 7 batch 100 loss 0.086089 acc 0.843750\n",
      "Epoch 7 batch 200 loss 0.103567 acc 0.812500\n",
      "Epoch 7 batch 300 loss 0.076172 acc 0.875000\n",
      "Epoch 7 batch 400 loss 0.111277 acc 0.781250\n",
      "Epoch 7 batch 500 loss 0.071214 acc 0.906250\n",
      "Epoch 7 batch 600 loss 0.093699 acc 0.875000\n",
      "Epoch 7 batch 700 loss 0.089088 acc 0.843750\n",
      "Epoch 7 batch 800 loss 0.089478 acc 0.875000\n",
      "Epoch 7 batch 900 loss 0.074798 acc 0.875000\n",
      "Epoch 7 batch 1000 loss 0.081065 acc 0.843750\n",
      "Epoch 7 batch 1100 loss 0.104302 acc 0.781250\n",
      "Validation accuracy: 0.829742\n",
      "Epoch 8 batch 0 loss 0.067731 acc 0.937500\n",
      "Epoch 8 batch 100 loss 0.086194 acc 0.843750\n",
      "Epoch 8 batch 200 loss 0.101794 acc 0.812500\n",
      "Epoch 8 batch 300 loss 0.077368 acc 0.843750\n",
      "Epoch 8 batch 400 loss 0.110587 acc 0.781250\n",
      "Epoch 8 batch 500 loss 0.070927 acc 0.906250\n",
      "Epoch 8 batch 600 loss 0.096229 acc 0.875000\n",
      "Epoch 8 batch 700 loss 0.089083 acc 0.843750\n",
      "Epoch 8 batch 800 loss 0.088433 acc 0.875000\n",
      "Epoch 8 batch 900 loss 0.075345 acc 0.875000\n",
      "Epoch 8 batch 1000 loss 0.080676 acc 0.843750\n",
      "Epoch 8 batch 1100 loss 0.102450 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 9 batch 0 loss 0.067562 acc 0.937500\n",
      "Epoch 9 batch 100 loss 0.085922 acc 0.812500\n",
      "Epoch 9 batch 200 loss 0.101405 acc 0.812500\n",
      "Epoch 9 batch 300 loss 0.077768 acc 0.843750\n",
      "Epoch 9 batch 400 loss 0.111334 acc 0.781250\n",
      "Epoch 9 batch 500 loss 0.072299 acc 0.906250\n",
      "Epoch 9 batch 600 loss 0.099077 acc 0.875000\n",
      "Epoch 9 batch 700 loss 0.089730 acc 0.843750\n",
      "Epoch 9 batch 800 loss 0.086183 acc 0.875000\n",
      "Epoch 9 batch 900 loss 0.075897 acc 0.875000\n",
      "Epoch 9 batch 1000 loss 0.080853 acc 0.843750\n",
      "Epoch 9 batch 1100 loss 0.100501 acc 0.781250\n",
      "Validation accuracy: 0.830215\n",
      "Epoch 10 batch 0 loss 0.063188 acc 0.937500\n",
      "Epoch 10 batch 100 loss 0.085531 acc 0.812500\n",
      "Epoch 10 batch 200 loss 0.100952 acc 0.812500\n",
      "Epoch 10 batch 300 loss 0.079642 acc 0.843750\n",
      "Epoch 10 batch 400 loss 0.109493 acc 0.781250\n",
      "Epoch 10 batch 500 loss 0.074166 acc 0.906250\n",
      "Epoch 10 batch 600 loss 0.101427 acc 0.843750\n",
      "Epoch 10 batch 700 loss 0.089030 acc 0.843750\n",
      "Epoch 10 batch 800 loss 0.084406 acc 0.875000\n",
      "Epoch 10 batch 900 loss 0.075679 acc 0.875000\n",
      "Epoch 10 batch 1000 loss 0.080417 acc 0.843750\n",
      "Epoch 10 batch 1100 loss 0.098666 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 11 batch 0 loss 0.073390 acc 0.937500\n",
      "Epoch 11 batch 100 loss 0.085249 acc 0.812500\n",
      "Epoch 11 batch 200 loss 0.099644 acc 0.812500\n",
      "Epoch 11 batch 300 loss 0.082051 acc 0.843750\n",
      "Epoch 11 batch 400 loss 0.107822 acc 0.781250\n",
      "Epoch 11 batch 500 loss 0.074426 acc 0.906250\n",
      "Epoch 11 batch 600 loss 0.101688 acc 0.843750\n",
      "Epoch 11 batch 700 loss 0.087962 acc 0.843750\n",
      "Epoch 11 batch 800 loss 0.083159 acc 0.875000\n",
      "Epoch 11 batch 900 loss 0.074939 acc 0.875000\n",
      "Epoch 11 batch 1000 loss 0.079577 acc 0.843750\n",
      "Epoch 11 batch 1100 loss 0.098104 acc 0.781250\n",
      "Validation accuracy: 0.831242\n",
      "Epoch 12 batch 0 loss 0.049169 acc 0.937500\n",
      "Epoch 12 batch 100 loss 0.084817 acc 0.812500\n",
      "Epoch 12 batch 200 loss 0.098724 acc 0.812500\n",
      "Epoch 12 batch 300 loss 0.086280 acc 0.875000\n",
      "Epoch 12 batch 400 loss 0.107088 acc 0.781250\n",
      "Epoch 12 batch 500 loss 0.075893 acc 0.906250\n",
      "Epoch 12 batch 600 loss 0.099294 acc 0.843750\n",
      "Epoch 12 batch 700 loss 0.086889 acc 0.843750\n",
      "Epoch 12 batch 800 loss 0.082186 acc 0.875000\n",
      "Epoch 12 batch 900 loss 0.073674 acc 0.875000\n",
      "Epoch 12 batch 1000 loss 0.078745 acc 0.843750\n",
      "Epoch 12 batch 1100 loss 0.100763 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 13 batch 0 loss 0.043314 acc 0.937500\n",
      "Epoch 13 batch 100 loss 0.084597 acc 0.812500\n",
      "Epoch 13 batch 200 loss 0.097928 acc 0.812500\n",
      "Epoch 13 batch 300 loss 0.087386 acc 0.875000\n",
      "Epoch 13 batch 400 loss 0.106542 acc 0.781250\n",
      "Epoch 13 batch 500 loss 0.075887 acc 0.906250\n",
      "Epoch 13 batch 600 loss 0.097562 acc 0.843750\n",
      "Epoch 13 batch 700 loss 0.086733 acc 0.843750\n",
      "Epoch 13 batch 800 loss 0.081942 acc 0.875000\n",
      "Epoch 13 batch 900 loss 0.072904 acc 0.875000\n",
      "Epoch 13 batch 1000 loss 0.078243 acc 0.843750\n",
      "Epoch 13 batch 1100 loss 0.103471 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 14 batch 0 loss 0.042395 acc 0.937500\n",
      "Epoch 14 batch 100 loss 0.084488 acc 0.812500\n",
      "Epoch 14 batch 200 loss 0.097357 acc 0.812500\n",
      "Epoch 14 batch 300 loss 0.087280 acc 0.875000\n",
      "Epoch 14 batch 400 loss 0.105897 acc 0.781250\n",
      "Epoch 14 batch 500 loss 0.075972 acc 0.906250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14 batch 600 loss 0.097010 acc 0.843750\n",
      "Epoch 14 batch 700 loss 0.086997 acc 0.843750\n",
      "Epoch 14 batch 800 loss 0.081775 acc 0.875000\n",
      "Epoch 14 batch 900 loss 0.072508 acc 0.875000\n",
      "Epoch 14 batch 1000 loss 0.078148 acc 0.843750\n",
      "Epoch 14 batch 1100 loss 0.104319 acc 0.781250\n",
      "Validation accuracy: 0.829663\n",
      "Epoch 15 batch 0 loss 0.041701 acc 0.937500\n",
      "Epoch 15 batch 100 loss 0.084422 acc 0.812500\n",
      "Epoch 15 batch 200 loss 0.096896 acc 0.812500\n",
      "Epoch 15 batch 300 loss 0.086139 acc 0.875000\n",
      "Epoch 15 batch 400 loss 0.105654 acc 0.781250\n",
      "Epoch 15 batch 500 loss 0.076539 acc 0.906250\n",
      "Epoch 15 batch 600 loss 0.096697 acc 0.843750\n",
      "Epoch 15 batch 700 loss 0.087036 acc 0.843750\n",
      "Epoch 15 batch 800 loss 0.081471 acc 0.875000\n",
      "Epoch 15 batch 900 loss 0.072299 acc 0.875000\n",
      "Epoch 15 batch 1000 loss 0.078548 acc 0.843750\n",
      "Epoch 15 batch 1100 loss 0.104005 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 16 batch 0 loss 0.041221 acc 0.937500\n",
      "Epoch 16 batch 100 loss 0.084417 acc 0.812500\n",
      "Epoch 16 batch 200 loss 0.096582 acc 0.812500\n",
      "Epoch 16 batch 300 loss 0.084314 acc 0.875000\n",
      "Epoch 16 batch 400 loss 0.105652 acc 0.781250\n",
      "Epoch 16 batch 500 loss 0.077357 acc 0.906250\n",
      "Epoch 16 batch 600 loss 0.096169 acc 0.843750\n",
      "Epoch 16 batch 700 loss 0.087119 acc 0.843750\n",
      "Epoch 16 batch 800 loss 0.081191 acc 0.875000\n",
      "Epoch 16 batch 900 loss 0.072190 acc 0.875000\n",
      "Epoch 16 batch 1000 loss 0.079104 acc 0.843750\n",
      "Epoch 16 batch 1100 loss 0.103273 acc 0.781250\n",
      "Validation accuracy: 0.829110\n",
      "Epoch 17 batch 0 loss 0.040026 acc 0.937500\n",
      "Epoch 17 batch 100 loss 0.084435 acc 0.812500\n",
      "Epoch 17 batch 200 loss 0.096432 acc 0.812500\n",
      "Epoch 17 batch 300 loss 0.083787 acc 0.875000\n",
      "Epoch 17 batch 400 loss 0.105491 acc 0.781250\n",
      "Epoch 17 batch 500 loss 0.077466 acc 0.906250\n",
      "Epoch 17 batch 600 loss 0.095891 acc 0.843750\n",
      "Epoch 17 batch 700 loss 0.087896 acc 0.843750\n",
      "Epoch 17 batch 800 loss 0.081189 acc 0.875000\n",
      "Epoch 17 batch 900 loss 0.072149 acc 0.875000\n",
      "Epoch 17 batch 1000 loss 0.079276 acc 0.843750\n",
      "Epoch 17 batch 1100 loss 0.102856 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 18 batch 0 loss 0.040592 acc 0.937500\n",
      "Epoch 18 batch 100 loss 0.084506 acc 0.812500\n",
      "Epoch 18 batch 200 loss 0.096278 acc 0.812500\n",
      "Epoch 18 batch 300 loss 0.083530 acc 0.875000\n",
      "Epoch 18 batch 400 loss 0.105314 acc 0.781250\n",
      "Epoch 18 batch 500 loss 0.077783 acc 0.906250\n",
      "Epoch 18 batch 600 loss 0.095772 acc 0.843750\n",
      "Epoch 18 batch 700 loss 0.088705 acc 0.843750\n",
      "Epoch 18 batch 800 loss 0.081602 acc 0.875000\n",
      "Epoch 18 batch 900 loss 0.072127 acc 0.875000\n",
      "Epoch 18 batch 1000 loss 0.079372 acc 0.843750\n",
      "Epoch 18 batch 1100 loss 0.102962 acc 0.781250\n",
      "Validation accuracy: 0.829031\n",
      "Epoch 19 batch 0 loss 0.040368 acc 0.937500\n",
      "Epoch 19 batch 100 loss 0.084591 acc 0.812500\n",
      "Epoch 19 batch 200 loss 0.096179 acc 0.812500\n",
      "Epoch 19 batch 300 loss 0.083453 acc 0.875000\n",
      "Epoch 19 batch 400 loss 0.105113 acc 0.781250\n",
      "Epoch 19 batch 500 loss 0.078299 acc 0.906250\n",
      "Epoch 19 batch 600 loss 0.095847 acc 0.843750\n",
      "Epoch 19 batch 700 loss 0.088785 acc 0.843750\n",
      "Epoch 19 batch 800 loss 0.082133 acc 0.875000\n",
      "Epoch 19 batch 900 loss 0.072121 acc 0.875000\n",
      "Epoch 19 batch 1000 loss 0.079457 acc 0.843750\n",
      "Epoch 19 batch 1100 loss 0.103399 acc 0.781250\n",
      "Validation accuracy: 0.828953\n",
      "Epoch 20 batch 0 loss 0.039732 acc 0.937500\n",
      "Epoch 20 batch 100 loss 0.084645 acc 0.812500\n",
      "Epoch 20 batch 200 loss 0.096048 acc 0.812500\n",
      "Epoch 20 batch 300 loss 0.083540 acc 0.875000\n",
      "Epoch 20 batch 400 loss 0.104903 acc 0.812500\n",
      "Epoch 20 batch 500 loss 0.078425 acc 0.906250\n",
      "Epoch 20 batch 600 loss 0.096204 acc 0.843750\n",
      "Epoch 20 batch 700 loss 0.088988 acc 0.843750\n",
      "Epoch 20 batch 800 loss 0.081991 acc 0.875000\n",
      "Epoch 20 batch 900 loss 0.072099 acc 0.875000\n",
      "Epoch 20 batch 1000 loss 0.079549 acc 0.843750\n",
      "Epoch 20 batch 1100 loss 0.103655 acc 0.781250\n",
      "Validation accuracy: 0.829031\n",
      "Epoch 21 batch 0 loss 0.040187 acc 0.937500\n",
      "Epoch 21 batch 100 loss 0.084637 acc 0.812500\n",
      "Epoch 21 batch 200 loss 0.095889 acc 0.812500\n",
      "Epoch 21 batch 300 loss 0.083683 acc 0.875000\n",
      "Epoch 21 batch 400 loss 0.104764 acc 0.812500\n",
      "Epoch 21 batch 500 loss 0.077970 acc 0.906250\n",
      "Epoch 21 batch 600 loss 0.096619 acc 0.843750\n",
      "Epoch 21 batch 700 loss 0.090437 acc 0.843750\n",
      "Epoch 21 batch 800 loss 0.081609 acc 0.875000\n",
      "Epoch 21 batch 900 loss 0.072056 acc 0.875000\n",
      "Epoch 21 batch 1000 loss 0.079597 acc 0.843750\n",
      "Epoch 21 batch 1100 loss 0.103429 acc 0.781250\n",
      "Validation accuracy: 0.830531\n",
      "Epoch 22 batch 0 loss 0.043402 acc 0.937500\n",
      "Epoch 22 batch 100 loss 0.084612 acc 0.812500\n",
      "Epoch 22 batch 200 loss 0.095709 acc 0.812500\n",
      "Epoch 22 batch 300 loss 0.083263 acc 0.875000\n",
      "Epoch 22 batch 400 loss 0.104672 acc 0.812500\n",
      "Epoch 22 batch 500 loss 0.076910 acc 0.906250\n",
      "Epoch 22 batch 600 loss 0.097286 acc 0.843750\n",
      "Epoch 22 batch 700 loss 0.091556 acc 0.843750\n",
      "Epoch 22 batch 800 loss 0.081234 acc 0.875000\n",
      "Epoch 22 batch 900 loss 0.072152 acc 0.875000\n",
      "Epoch 22 batch 1000 loss 0.079611 acc 0.843750\n",
      "Epoch 22 batch 1100 loss 0.103093 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 23 batch 0 loss 0.041857 acc 0.937500\n",
      "Epoch 23 batch 100 loss 0.084647 acc 0.812500\n",
      "Epoch 23 batch 200 loss 0.095592 acc 0.812500\n",
      "Epoch 23 batch 300 loss 0.083105 acc 0.875000\n",
      "Epoch 23 batch 400 loss 0.104706 acc 0.812500\n",
      "Epoch 23 batch 500 loss 0.076649 acc 0.906250\n",
      "Epoch 23 batch 600 loss 0.097340 acc 0.843750\n",
      "Epoch 23 batch 700 loss 0.092221 acc 0.843750\n",
      "Epoch 23 batch 800 loss 0.081064 acc 0.875000\n",
      "Epoch 23 batch 900 loss 0.072200 acc 0.875000\n",
      "Epoch 23 batch 1000 loss 0.079590 acc 0.843750\n",
      "Epoch 23 batch 1100 loss 0.102837 acc 0.781250\n",
      "Validation accuracy: 0.829584\n",
      "Epoch 24 batch 0 loss 0.041735 acc 0.937500\n",
      "Epoch 24 batch 100 loss 0.084645 acc 0.812500\n",
      "Epoch 24 batch 200 loss 0.095539 acc 0.812500\n",
      "Epoch 24 batch 300 loss 0.082944 acc 0.875000\n",
      "Epoch 24 batch 400 loss 0.104718 acc 0.812500\n",
      "Epoch 24 batch 500 loss 0.076439 acc 0.906250\n",
      "Epoch 24 batch 600 loss 0.097283 acc 0.843750\n",
      "Epoch 24 batch 700 loss 0.092550 acc 0.843750\n",
      "Epoch 24 batch 800 loss 0.080915 acc 0.875000\n",
      "Epoch 24 batch 900 loss 0.072226 acc 0.875000\n",
      "Epoch 24 batch 1000 loss 0.079539 acc 0.843750\n",
      "Epoch 24 batch 1100 loss 0.102691 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 25 batch 0 loss 0.041708 acc 0.937500\n",
      "Epoch 25 batch 100 loss 0.084650 acc 0.812500\n",
      "Epoch 25 batch 200 loss 0.095501 acc 0.812500\n",
      "Epoch 25 batch 300 loss 0.082834 acc 0.875000\n",
      "Epoch 25 batch 400 loss 0.104667 acc 0.812500\n",
      "Epoch 25 batch 500 loss 0.076227 acc 0.906250\n",
      "Epoch 25 batch 600 loss 0.097126 acc 0.843750\n",
      "Epoch 25 batch 700 loss 0.092483 acc 0.843750\n",
      "Epoch 25 batch 800 loss 0.080769 acc 0.875000\n",
      "Epoch 25 batch 900 loss 0.072237 acc 0.875000\n",
      "Epoch 25 batch 1000 loss 0.079487 acc 0.843750\n",
      "Epoch 25 batch 1100 loss 0.102568 acc 0.781250\n",
      "Validation accuracy: 0.829742\n",
      "Epoch 26 batch 0 loss 0.041940 acc 0.937500\n",
      "Epoch 26 batch 100 loss 0.084657 acc 0.812500\n",
      "Epoch 26 batch 200 loss 0.095462 acc 0.812500\n",
      "Epoch 26 batch 300 loss 0.082872 acc 0.875000\n",
      "Epoch 26 batch 400 loss 0.104593 acc 0.812500\n",
      "Epoch 26 batch 500 loss 0.075947 acc 0.906250\n",
      "Epoch 26 batch 600 loss 0.096913 acc 0.843750\n",
      "Epoch 26 batch 700 loss 0.092339 acc 0.843750\n",
      "Epoch 26 batch 800 loss 0.080667 acc 0.875000\n",
      "Epoch 26 batch 900 loss 0.072232 acc 0.875000\n",
      "Epoch 26 batch 1000 loss 0.079428 acc 0.843750\n",
      "Epoch 26 batch 1100 loss 0.102404 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 27 batch 0 loss 0.042080 acc 0.937500\n",
      "Epoch 27 batch 100 loss 0.084646 acc 0.812500\n",
      "Epoch 27 batch 200 loss 0.095418 acc 0.812500\n",
      "Epoch 27 batch 300 loss 0.083698 acc 0.875000\n",
      "Epoch 27 batch 400 loss 0.104550 acc 0.812500\n",
      "Epoch 27 batch 500 loss 0.075878 acc 0.906250\n",
      "Epoch 27 batch 600 loss 0.096682 acc 0.843750\n",
      "Epoch 27 batch 700 loss 0.092154 acc 0.843750\n",
      "Epoch 27 batch 800 loss 0.080793 acc 0.875000\n",
      "Epoch 27 batch 900 loss 0.072212 acc 0.875000\n",
      "Epoch 27 batch 1000 loss 0.079359 acc 0.843750\n",
      "Epoch 27 batch 1100 loss 0.102135 acc 0.781250\n",
      "Validation accuracy: 0.830610\n",
      "Epoch 28 batch 0 loss 0.042281 acc 0.937500\n",
      "Epoch 28 batch 100 loss 0.084504 acc 0.812500\n",
      "Epoch 28 batch 200 loss 0.095330 acc 0.812500\n",
      "Epoch 28 batch 300 loss 0.084557 acc 0.875000\n",
      "Epoch 28 batch 400 loss 0.104464 acc 0.812500\n",
      "Epoch 28 batch 500 loss 0.075466 acc 0.906250\n",
      "Epoch 28 batch 600 loss 0.096474 acc 0.843750\n",
      "Epoch 28 batch 700 loss 0.089497 acc 0.843750\n",
      "Epoch 28 batch 800 loss 0.081574 acc 0.875000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28 batch 900 loss 0.072211 acc 0.875000\n",
      "Epoch 28 batch 1000 loss 0.079279 acc 0.843750\n",
      "Epoch 28 batch 1100 loss 0.101999 acc 0.781250\n",
      "Validation accuracy: 0.829426\n",
      "Epoch 29 batch 0 loss 0.040774 acc 0.937500\n",
      "Epoch 29 batch 100 loss 0.084220 acc 0.812500\n",
      "Epoch 29 batch 200 loss 0.095173 acc 0.812500\n",
      "Epoch 29 batch 300 loss 0.084571 acc 0.875000\n",
      "Epoch 29 batch 400 loss 0.104428 acc 0.812500\n",
      "Epoch 29 batch 500 loss 0.074782 acc 0.906250\n",
      "Epoch 29 batch 600 loss 0.096529 acc 0.843750\n",
      "Epoch 29 batch 700 loss 0.090175 acc 0.843750\n",
      "Epoch 29 batch 800 loss 0.082007 acc 0.875000\n",
      "Epoch 29 batch 900 loss 0.072304 acc 0.875000\n",
      "Epoch 29 batch 1000 loss 0.079258 acc 0.843750\n",
      "Epoch 29 batch 1100 loss 0.101833 acc 0.781250\n",
      "Validation accuracy: 0.829426\n",
      "Epoch 30 batch 0 loss 0.040511 acc 0.937500\n",
      "Epoch 30 batch 100 loss 0.084252 acc 0.812500\n",
      "Epoch 30 batch 200 loss 0.095140 acc 0.812500\n",
      "Epoch 30 batch 300 loss 0.083770 acc 0.875000\n",
      "Epoch 30 batch 400 loss 0.104446 acc 0.812500\n",
      "Epoch 30 batch 500 loss 0.073739 acc 0.906250\n",
      "Epoch 30 batch 600 loss 0.096820 acc 0.843750\n",
      "Epoch 30 batch 700 loss 0.088650 acc 0.843750\n",
      "Epoch 30 batch 800 loss 0.081119 acc 0.875000\n",
      "Epoch 30 batch 900 loss 0.072427 acc 0.875000\n",
      "Epoch 30 batch 1000 loss 0.079570 acc 0.843750\n",
      "Epoch 30 batch 1100 loss 0.102043 acc 0.781250\n",
      "Validation accuracy: 0.828874\n",
      "Epoch 31 batch 0 loss 0.040076 acc 0.937500\n",
      "Epoch 31 batch 100 loss 0.084269 acc 0.812500\n",
      "Epoch 31 batch 200 loss 0.095093 acc 0.812500\n",
      "Epoch 31 batch 300 loss 0.085508 acc 0.875000\n",
      "Epoch 31 batch 400 loss 0.104514 acc 0.812500\n",
      "Epoch 31 batch 500 loss 0.073497 acc 0.906250\n",
      "Epoch 31 batch 600 loss 0.096960 acc 0.843750\n",
      "Epoch 31 batch 700 loss 0.089642 acc 0.843750\n",
      "Epoch 31 batch 800 loss 0.081039 acc 0.875000\n",
      "Epoch 31 batch 900 loss 0.072435 acc 0.875000\n",
      "Epoch 31 batch 1000 loss 0.079650 acc 0.843750\n",
      "Epoch 31 batch 1100 loss 0.101987 acc 0.781250\n",
      "Validation accuracy: 0.828874\n",
      "Epoch 32 batch 0 loss 0.039663 acc 0.937500\n",
      "Epoch 32 batch 100 loss 0.084174 acc 0.812500\n",
      "Epoch 32 batch 200 loss 0.095045 acc 0.812500\n",
      "Epoch 32 batch 300 loss 0.085186 acc 0.875000\n",
      "Epoch 32 batch 400 loss 0.104480 acc 0.812500\n",
      "Epoch 32 batch 500 loss 0.073458 acc 0.906250\n",
      "Epoch 32 batch 600 loss 0.097032 acc 0.843750\n",
      "Epoch 32 batch 700 loss 0.089410 acc 0.843750\n",
      "Epoch 32 batch 800 loss 0.080853 acc 0.875000\n",
      "Epoch 32 batch 900 loss 0.072434 acc 0.875000\n",
      "Epoch 32 batch 1000 loss 0.079684 acc 0.843750\n",
      "Epoch 32 batch 1100 loss 0.101906 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 33 batch 0 loss 0.040795 acc 0.937500\n",
      "Epoch 33 batch 100 loss 0.084079 acc 0.812500\n",
      "Epoch 33 batch 200 loss 0.095008 acc 0.812500\n",
      "Epoch 33 batch 300 loss 0.084631 acc 0.875000\n",
      "Epoch 33 batch 400 loss 0.104404 acc 0.812500\n",
      "Epoch 33 batch 500 loss 0.073354 acc 0.906250\n",
      "Epoch 33 batch 600 loss 0.097054 acc 0.843750\n",
      "Epoch 33 batch 700 loss 0.088989 acc 0.843750\n",
      "Epoch 33 batch 800 loss 0.080533 acc 0.875000\n",
      "Epoch 33 batch 900 loss 0.072500 acc 0.875000\n",
      "Epoch 33 batch 1000 loss 0.079728 acc 0.843750\n",
      "Epoch 33 batch 1100 loss 0.101792 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 34 batch 0 loss 0.041250 acc 0.937500\n",
      "Epoch 34 batch 100 loss 0.084034 acc 0.812500\n",
      "Epoch 34 batch 200 loss 0.094990 acc 0.812500\n",
      "Epoch 34 batch 300 loss 0.083111 acc 0.875000\n",
      "Epoch 34 batch 400 loss 0.103965 acc 0.812500\n",
      "Epoch 34 batch 500 loss 0.073027 acc 0.906250\n",
      "Epoch 34 batch 600 loss 0.097125 acc 0.843750\n",
      "Epoch 34 batch 700 loss 0.088469 acc 0.843750\n",
      "Epoch 34 batch 800 loss 0.080244 acc 0.875000\n",
      "Epoch 34 batch 900 loss 0.072572 acc 0.875000\n",
      "Epoch 34 batch 1000 loss 0.079748 acc 0.843750\n",
      "Epoch 34 batch 1100 loss 0.101645 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 35 batch 0 loss 0.041500 acc 0.937500\n",
      "Epoch 35 batch 100 loss 0.084011 acc 0.812500\n",
      "Epoch 35 batch 200 loss 0.094982 acc 0.812500\n",
      "Epoch 35 batch 300 loss 0.081343 acc 0.875000\n",
      "Epoch 35 batch 400 loss 0.103934 acc 0.812500\n",
      "Epoch 35 batch 500 loss 0.072869 acc 0.906250\n",
      "Epoch 35 batch 600 loss 0.097158 acc 0.843750\n",
      "Epoch 35 batch 700 loss 0.088124 acc 0.843750\n",
      "Epoch 35 batch 800 loss 0.080088 acc 0.875000\n",
      "Epoch 35 batch 900 loss 0.072601 acc 0.875000\n",
      "Epoch 35 batch 1000 loss 0.079759 acc 0.843750\n",
      "Epoch 35 batch 1100 loss 0.101533 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 36 batch 0 loss 0.041330 acc 0.937500\n",
      "Epoch 36 batch 100 loss 0.083993 acc 0.812500\n",
      "Epoch 36 batch 200 loss 0.094969 acc 0.812500\n",
      "Epoch 36 batch 300 loss 0.081202 acc 0.875000\n",
      "Epoch 36 batch 400 loss 0.103902 acc 0.812500\n",
      "Epoch 36 batch 500 loss 0.072723 acc 0.906250\n",
      "Epoch 36 batch 600 loss 0.097197 acc 0.843750\n",
      "Epoch 36 batch 700 loss 0.087962 acc 0.843750\n",
      "Epoch 36 batch 800 loss 0.080011 acc 0.875000\n",
      "Epoch 36 batch 900 loss 0.072610 acc 0.875000\n",
      "Epoch 36 batch 1000 loss 0.079766 acc 0.843750\n",
      "Epoch 36 batch 1100 loss 0.101404 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 37 batch 0 loss 0.041243 acc 0.937500\n",
      "Epoch 37 batch 100 loss 0.083972 acc 0.812500\n",
      "Epoch 37 batch 200 loss 0.094949 acc 0.812500\n",
      "Epoch 37 batch 300 loss 0.081088 acc 0.875000\n",
      "Epoch 37 batch 400 loss 0.103884 acc 0.812500\n",
      "Epoch 37 batch 500 loss 0.072591 acc 0.906250\n",
      "Epoch 37 batch 600 loss 0.097251 acc 0.843750\n",
      "Epoch 37 batch 700 loss 0.087845 acc 0.843750\n",
      "Epoch 37 batch 800 loss 0.079965 acc 0.875000\n",
      "Epoch 37 batch 900 loss 0.072606 acc 0.875000\n",
      "Epoch 37 batch 1000 loss 0.079773 acc 0.843750\n",
      "Epoch 37 batch 1100 loss 0.101258 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 38 batch 0 loss 0.041145 acc 0.937500\n",
      "Epoch 38 batch 100 loss 0.083950 acc 0.812500\n",
      "Epoch 38 batch 200 loss 0.094921 acc 0.812500\n",
      "Epoch 38 batch 300 loss 0.081009 acc 0.875000\n",
      "Epoch 38 batch 400 loss 0.103874 acc 0.812500\n",
      "Epoch 38 batch 500 loss 0.072472 acc 0.906250\n",
      "Epoch 38 batch 600 loss 0.097312 acc 0.843750\n",
      "Epoch 38 batch 700 loss 0.087757 acc 0.843750\n",
      "Epoch 38 batch 800 loss 0.079936 acc 0.875000\n",
      "Epoch 38 batch 900 loss 0.072592 acc 0.875000\n",
      "Epoch 38 batch 1000 loss 0.079781 acc 0.843750\n",
      "Epoch 38 batch 1100 loss 0.101099 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 39 batch 0 loss 0.041059 acc 0.937500\n",
      "Epoch 39 batch 100 loss 0.083928 acc 0.812500\n",
      "Epoch 39 batch 200 loss 0.094884 acc 0.812500\n",
      "Epoch 39 batch 300 loss 0.080960 acc 0.875000\n",
      "Epoch 39 batch 400 loss 0.103885 acc 0.812500\n",
      "Epoch 39 batch 500 loss 0.072370 acc 0.906250\n",
      "Epoch 39 batch 600 loss 0.097370 acc 0.843750\n",
      "Epoch 39 batch 700 loss 0.087685 acc 0.843750\n",
      "Epoch 39 batch 800 loss 0.079912 acc 0.875000\n",
      "Epoch 39 batch 900 loss 0.072572 acc 0.875000\n",
      "Epoch 39 batch 1000 loss 0.079787 acc 0.843750\n",
      "Epoch 39 batch 1100 loss 0.100931 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 40 batch 0 loss 0.040956 acc 0.937500\n",
      "Epoch 40 batch 100 loss 0.083910 acc 0.812500\n",
      "Epoch 40 batch 200 loss 0.094840 acc 0.812500\n",
      "Epoch 40 batch 300 loss 0.080939 acc 0.875000\n",
      "Epoch 40 batch 400 loss 0.103899 acc 0.812500\n",
      "Epoch 40 batch 500 loss 0.072278 acc 0.906250\n",
      "Epoch 40 batch 600 loss 0.097424 acc 0.843750\n",
      "Epoch 40 batch 700 loss 0.087627 acc 0.843750\n",
      "Epoch 40 batch 800 loss 0.079894 acc 0.875000\n",
      "Epoch 40 batch 900 loss 0.072548 acc 0.875000\n",
      "Epoch 40 batch 1000 loss 0.079792 acc 0.843750\n",
      "Epoch 40 batch 1100 loss 0.100760 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 41 batch 0 loss 0.040848 acc 0.937500\n",
      "Epoch 41 batch 100 loss 0.083899 acc 0.812500\n",
      "Epoch 41 batch 200 loss 0.094792 acc 0.812500\n",
      "Epoch 41 batch 300 loss 0.080951 acc 0.875000\n",
      "Epoch 41 batch 400 loss 0.103889 acc 0.812500\n",
      "Epoch 41 batch 500 loss 0.072184 acc 0.906250\n",
      "Epoch 41 batch 600 loss 0.097480 acc 0.843750\n",
      "Epoch 41 batch 700 loss 0.087579 acc 0.843750\n",
      "Epoch 41 batch 800 loss 0.079882 acc 0.875000\n",
      "Epoch 41 batch 900 loss 0.072523 acc 0.875000\n",
      "Epoch 41 batch 1000 loss 0.079794 acc 0.843750\n",
      "Epoch 41 batch 1100 loss 0.100593 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 42 batch 0 loss 0.040751 acc 0.937500\n",
      "Epoch 42 batch 100 loss 0.083896 acc 0.843750\n",
      "Epoch 42 batch 200 loss 0.094743 acc 0.812500\n",
      "Epoch 42 batch 300 loss 0.081004 acc 0.875000\n",
      "Epoch 42 batch 400 loss 0.103835 acc 0.812500\n",
      "Epoch 42 batch 500 loss 0.072074 acc 0.906250\n",
      "Epoch 42 batch 600 loss 0.097547 acc 0.843750\n",
      "Epoch 42 batch 700 loss 0.087536 acc 0.843750\n",
      "Epoch 42 batch 800 loss 0.079879 acc 0.875000\n",
      "Epoch 42 batch 900 loss 0.072502 acc 0.875000\n",
      "Epoch 42 batch 1000 loss 0.079793 acc 0.843750\n",
      "Epoch 42 batch 1100 loss 0.100439 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.829426\n",
      "Epoch 43 batch 0 loss 0.040702 acc 0.937500\n",
      "Epoch 43 batch 100 loss 0.083902 acc 0.843750\n",
      "Epoch 43 batch 200 loss 0.094695 acc 0.812500\n",
      "Epoch 43 batch 300 loss 0.081130 acc 0.875000\n",
      "Epoch 43 batch 400 loss 0.103757 acc 0.812500\n",
      "Epoch 43 batch 500 loss 0.071947 acc 0.906250\n",
      "Epoch 43 batch 600 loss 0.097631 acc 0.843750\n",
      "Epoch 43 batch 700 loss 0.087499 acc 0.843750\n",
      "Epoch 43 batch 800 loss 0.079884 acc 0.875000\n",
      "Epoch 43 batch 900 loss 0.072486 acc 0.875000\n",
      "Epoch 43 batch 1000 loss 0.079789 acc 0.843750\n",
      "Epoch 43 batch 1100 loss 0.100296 acc 0.781250\n",
      "Validation accuracy: 0.829426\n",
      "Epoch 44 batch 0 loss 0.040709 acc 0.937500\n",
      "Epoch 44 batch 100 loss 0.083919 acc 0.843750\n",
      "Epoch 44 batch 200 loss 0.094650 acc 0.812500\n",
      "Epoch 44 batch 300 loss 0.081373 acc 0.875000\n",
      "Epoch 44 batch 400 loss 0.103682 acc 0.812500\n",
      "Epoch 44 batch 500 loss 0.071807 acc 0.906250\n",
      "Epoch 44 batch 600 loss 0.097735 acc 0.843750\n",
      "Epoch 44 batch 700 loss 0.087474 acc 0.843750\n",
      "Epoch 44 batch 800 loss 0.079898 acc 0.875000\n",
      "Epoch 44 batch 900 loss 0.072478 acc 0.875000\n",
      "Epoch 44 batch 1000 loss 0.079781 acc 0.843750\n",
      "Epoch 44 batch 1100 loss 0.100163 acc 0.781250\n",
      "Validation accuracy: 0.829426\n",
      "Epoch 45 batch 0 loss 0.040779 acc 0.937500\n",
      "Epoch 45 batch 100 loss 0.083944 acc 0.843750\n",
      "Epoch 45 batch 200 loss 0.094609 acc 0.812500\n",
      "Epoch 45 batch 300 loss 0.081767 acc 0.875000\n",
      "Epoch 45 batch 400 loss 0.103627 acc 0.812500\n",
      "Epoch 45 batch 500 loss 0.071666 acc 0.906250\n",
      "Epoch 45 batch 600 loss 0.097856 acc 0.843750\n",
      "Epoch 45 batch 700 loss 0.087465 acc 0.843750\n",
      "Epoch 45 batch 800 loss 0.079915 acc 0.875000\n",
      "Epoch 45 batch 900 loss 0.072478 acc 0.875000\n",
      "Epoch 45 batch 1000 loss 0.079771 acc 0.843750\n",
      "Epoch 45 batch 1100 loss 0.100038 acc 0.781250\n",
      "Validation accuracy: 0.829584\n",
      "Epoch 46 batch 0 loss 0.040933 acc 0.937500\n",
      "Epoch 46 batch 100 loss 0.083972 acc 0.843750\n",
      "Epoch 46 batch 200 loss 0.094572 acc 0.812500\n",
      "Epoch 46 batch 300 loss 0.082310 acc 0.875000\n",
      "Epoch 46 batch 400 loss 0.103585 acc 0.812500\n",
      "Epoch 46 batch 500 loss 0.071531 acc 0.906250\n",
      "Epoch 46 batch 600 loss 0.097994 acc 0.843750\n",
      "Epoch 46 batch 700 loss 0.087472 acc 0.843750\n",
      "Epoch 46 batch 800 loss 0.079930 acc 0.875000\n",
      "Epoch 46 batch 900 loss 0.072487 acc 0.875000\n",
      "Epoch 46 batch 1000 loss 0.079760 acc 0.843750\n",
      "Epoch 46 batch 1100 loss 0.099919 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 47 batch 0 loss 0.041189 acc 0.937500\n",
      "Epoch 47 batch 100 loss 0.083995 acc 0.843750\n",
      "Epoch 47 batch 200 loss 0.094538 acc 0.812500\n",
      "Epoch 47 batch 300 loss 0.083000 acc 0.875000\n",
      "Epoch 47 batch 400 loss 0.103543 acc 0.812500\n",
      "Epoch 47 batch 500 loss 0.071402 acc 0.906250\n",
      "Epoch 47 batch 600 loss 0.098149 acc 0.843750\n",
      "Epoch 47 batch 700 loss 0.087500 acc 0.843750\n",
      "Epoch 47 batch 800 loss 0.079940 acc 0.875000\n",
      "Epoch 47 batch 900 loss 0.072503 acc 0.875000\n",
      "Epoch 47 batch 1000 loss 0.079753 acc 0.843750\n",
      "Epoch 47 batch 1100 loss 0.099809 acc 0.781250\n",
      "Validation accuracy: 0.829584\n",
      "Epoch 48 batch 0 loss 0.041559 acc 0.937500\n",
      "Epoch 48 batch 100 loss 0.084005 acc 0.843750\n",
      "Epoch 48 batch 200 loss 0.094506 acc 0.812500\n",
      "Epoch 48 batch 300 loss 0.083766 acc 0.875000\n",
      "Epoch 48 batch 400 loss 0.103499 acc 0.812500\n",
      "Epoch 48 batch 500 loss 0.071296 acc 0.906250\n",
      "Epoch 48 batch 600 loss 0.098317 acc 0.843750\n",
      "Epoch 48 batch 700 loss 0.087566 acc 0.843750\n",
      "Epoch 48 batch 800 loss 0.079947 acc 0.875000\n",
      "Epoch 48 batch 900 loss 0.072521 acc 0.875000\n",
      "Epoch 48 batch 1000 loss 0.079748 acc 0.843750\n",
      "Epoch 48 batch 1100 loss 0.099692 acc 0.781250\n",
      "Validation accuracy: 0.829979\n",
      "Epoch 49 batch 0 loss 0.042126 acc 0.937500\n",
      "Epoch 49 batch 100 loss 0.083985 acc 0.843750\n",
      "Epoch 49 batch 200 loss 0.094475 acc 0.812500\n",
      "Epoch 49 batch 300 loss 0.084469 acc 0.875000\n",
      "Epoch 49 batch 400 loss 0.103459 acc 0.812500\n",
      "Epoch 49 batch 500 loss 0.071344 acc 0.906250\n",
      "Epoch 49 batch 600 loss 0.098449 acc 0.843750\n",
      "Epoch 49 batch 700 loss 0.087602 acc 0.843750\n",
      "Epoch 49 batch 800 loss 0.079945 acc 0.875000\n",
      "Epoch 49 batch 900 loss 0.072525 acc 0.875000\n",
      "Epoch 49 batch 1000 loss 0.079741 acc 0.843750\n",
      "Epoch 49 batch 1100 loss 0.099615 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 50 batch 0 loss 0.042682 acc 0.937500\n",
      "Epoch 50 batch 100 loss 0.083898 acc 0.843750\n",
      "Epoch 50 batch 200 loss 0.094433 acc 0.812500\n",
      "Epoch 50 batch 300 loss 0.084955 acc 0.875000\n",
      "Epoch 50 batch 400 loss 0.103521 acc 0.812500\n",
      "Epoch 50 batch 500 loss 0.071534 acc 0.906250\n",
      "Epoch 50 batch 600 loss 0.098532 acc 0.843750\n",
      "Epoch 50 batch 700 loss 0.087474 acc 0.843750\n",
      "Epoch 50 batch 800 loss 0.079935 acc 0.875000\n",
      "Epoch 50 batch 900 loss 0.072505 acc 0.875000\n",
      "Epoch 50 batch 1000 loss 0.079719 acc 0.843750\n",
      "Epoch 50 batch 1100 loss 0.099773 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 51 batch 0 loss 0.042564 acc 0.937500\n",
      "Epoch 51 batch 100 loss 0.083827 acc 0.843750\n",
      "Epoch 51 batch 200 loss 0.094380 acc 0.812500\n",
      "Epoch 51 batch 300 loss 0.085877 acc 0.875000\n",
      "Epoch 51 batch 400 loss 0.103556 acc 0.812500\n",
      "Epoch 51 batch 500 loss 0.071414 acc 0.906250\n",
      "Epoch 51 batch 600 loss 0.098690 acc 0.843750\n",
      "Epoch 51 batch 700 loss 0.087658 acc 0.843750\n",
      "Epoch 51 batch 800 loss 0.079905 acc 0.875000\n",
      "Epoch 51 batch 900 loss 0.072543 acc 0.875000\n",
      "Epoch 51 batch 1000 loss 0.079699 acc 0.843750\n",
      "Epoch 51 batch 1100 loss 0.099607 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 52 batch 0 loss 0.041603 acc 0.937500\n",
      "Epoch 52 batch 100 loss 0.083968 acc 0.843750\n",
      "Epoch 52 batch 200 loss 0.094358 acc 0.812500\n",
      "Epoch 52 batch 300 loss 0.087312 acc 0.875000\n",
      "Epoch 52 batch 400 loss 0.103531 acc 0.812500\n",
      "Epoch 52 batch 500 loss 0.071415 acc 0.906250\n",
      "Epoch 52 batch 600 loss 0.098899 acc 0.843750\n",
      "Epoch 52 batch 700 loss 0.088090 acc 0.843750\n",
      "Epoch 52 batch 800 loss 0.079866 acc 0.875000\n",
      "Epoch 52 batch 900 loss 0.072595 acc 0.875000\n",
      "Epoch 52 batch 1000 loss 0.079692 acc 0.843750\n",
      "Epoch 52 batch 1100 loss 0.098224 acc 0.781250\n",
      "Validation accuracy: 0.831242\n",
      "Epoch 53 batch 0 loss 0.044321 acc 0.937500\n",
      "Epoch 53 batch 100 loss 0.083619 acc 0.843750\n",
      "Epoch 53 batch 200 loss 0.094331 acc 0.812500\n",
      "Epoch 53 batch 300 loss 0.085664 acc 0.875000\n",
      "Epoch 53 batch 400 loss 0.103386 acc 0.812500\n",
      "Epoch 53 batch 500 loss 0.071526 acc 0.906250\n",
      "Epoch 53 batch 600 loss 0.098992 acc 0.843750\n",
      "Epoch 53 batch 700 loss 0.087960 acc 0.843750\n",
      "Epoch 53 batch 800 loss 0.079929 acc 0.875000\n",
      "Epoch 53 batch 900 loss 0.072615 acc 0.875000\n",
      "Epoch 53 batch 1000 loss 0.079573 acc 0.843750\n",
      "Epoch 53 batch 1100 loss 0.098263 acc 0.781250\n",
      "Validation accuracy: 0.831242\n",
      "Epoch 54 batch 0 loss 0.044422 acc 0.937500\n",
      "Epoch 54 batch 100 loss 0.083748 acc 0.843750\n",
      "Epoch 54 batch 200 loss 0.094325 acc 0.812500\n",
      "Epoch 54 batch 300 loss 0.086243 acc 0.875000\n",
      "Epoch 54 batch 400 loss 0.103427 acc 0.812500\n",
      "Epoch 54 batch 500 loss 0.070906 acc 0.906250\n",
      "Epoch 54 batch 600 loss 0.099237 acc 0.843750\n",
      "Epoch 54 batch 700 loss 0.088193 acc 0.843750\n",
      "Epoch 54 batch 800 loss 0.079793 acc 0.875000\n",
      "Epoch 54 batch 900 loss 0.072617 acc 0.875000\n",
      "Epoch 54 batch 1000 loss 0.079606 acc 0.843750\n",
      "Epoch 54 batch 1100 loss 0.098649 acc 0.781250\n",
      "Validation accuracy: 0.830452\n",
      "Epoch 55 batch 0 loss 0.043701 acc 0.937500\n",
      "Epoch 55 batch 100 loss 0.083681 acc 0.843750\n",
      "Epoch 55 batch 200 loss 0.094279 acc 0.812500\n",
      "Epoch 55 batch 300 loss 0.086357 acc 0.875000\n",
      "Epoch 55 batch 400 loss 0.103365 acc 0.812500\n",
      "Epoch 55 batch 500 loss 0.071097 acc 0.906250\n",
      "Epoch 55 batch 600 loss 0.099371 acc 0.843750\n",
      "Epoch 55 batch 700 loss 0.088210 acc 0.843750\n",
      "Epoch 55 batch 800 loss 0.079770 acc 0.875000\n",
      "Epoch 55 batch 900 loss 0.072611 acc 0.875000\n",
      "Epoch 55 batch 1000 loss 0.079545 acc 0.843750\n",
      "Epoch 55 batch 1100 loss 0.099611 acc 0.781250\n",
      "Validation accuracy: 0.830215\n",
      "Epoch 56 batch 0 loss 0.041887 acc 0.937500\n",
      "Epoch 56 batch 100 loss 0.083882 acc 0.843750\n",
      "Epoch 56 batch 200 loss 0.094267 acc 0.812500\n",
      "Epoch 56 batch 300 loss 0.085438 acc 0.875000\n",
      "Epoch 56 batch 400 loss 0.103433 acc 0.812500\n",
      "Epoch 56 batch 500 loss 0.072117 acc 0.906250\n",
      "Epoch 56 batch 600 loss 0.099372 acc 0.843750\n",
      "Epoch 56 batch 700 loss 0.088141 acc 0.843750\n",
      "Epoch 56 batch 800 loss 0.079973 acc 0.875000\n",
      "Epoch 56 batch 900 loss 0.072574 acc 0.875000\n",
      "Epoch 56 batch 1000 loss 0.079449 acc 0.843750\n",
      "Epoch 56 batch 1100 loss 0.099256 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 57 batch 0 loss 0.042061 acc 0.937500\n",
      "Epoch 57 batch 100 loss 0.083486 acc 0.843750\n",
      "Epoch 57 batch 200 loss 0.094158 acc 0.812500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57 batch 300 loss 0.084881 acc 0.875000\n",
      "Epoch 57 batch 400 loss 0.103392 acc 0.812500\n",
      "Epoch 57 batch 500 loss 0.072581 acc 0.906250\n",
      "Epoch 57 batch 600 loss 0.099462 acc 0.843750\n",
      "Epoch 57 batch 700 loss 0.088525 acc 0.843750\n",
      "Epoch 57 batch 800 loss 0.079745 acc 0.875000\n",
      "Epoch 57 batch 900 loss 0.072599 acc 0.875000\n",
      "Epoch 57 batch 1000 loss 0.079630 acc 0.843750\n",
      "Epoch 57 batch 1100 loss 0.098335 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 58 batch 0 loss 0.040335 acc 0.937500\n",
      "Epoch 58 batch 100 loss 0.083486 acc 0.843750\n",
      "Epoch 58 batch 200 loss 0.094038 acc 0.812500\n",
      "Epoch 58 batch 300 loss 0.084542 acc 0.875000\n",
      "Epoch 58 batch 400 loss 0.103397 acc 0.812500\n",
      "Epoch 58 batch 500 loss 0.072926 acc 0.906250\n",
      "Epoch 58 batch 600 loss 0.099581 acc 0.843750\n",
      "Epoch 58 batch 700 loss 0.088580 acc 0.843750\n",
      "Epoch 58 batch 800 loss 0.079618 acc 0.875000\n",
      "Epoch 58 batch 900 loss 0.072614 acc 0.875000\n",
      "Epoch 58 batch 1000 loss 0.079580 acc 0.843750\n",
      "Epoch 58 batch 1100 loss 0.099699 acc 0.781250\n",
      "Validation accuracy: 0.829426\n",
      "Epoch 59 batch 0 loss 0.040748 acc 0.937500\n",
      "Epoch 59 batch 100 loss 0.083508 acc 0.843750\n",
      "Epoch 59 batch 200 loss 0.094040 acc 0.812500\n",
      "Epoch 59 batch 300 loss 0.083153 acc 0.875000\n",
      "Epoch 59 batch 400 loss 0.103263 acc 0.812500\n",
      "Epoch 59 batch 500 loss 0.073123 acc 0.906250\n",
      "Epoch 59 batch 600 loss 0.099648 acc 0.843750\n",
      "Epoch 59 batch 700 loss 0.088626 acc 0.843750\n",
      "Epoch 59 batch 800 loss 0.079501 acc 0.875000\n",
      "Epoch 59 batch 900 loss 0.072608 acc 0.875000\n",
      "Epoch 59 batch 1000 loss 0.079759 acc 0.843750\n",
      "Epoch 59 batch 1100 loss 0.097459 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 60 batch 0 loss 0.040622 acc 0.937500\n",
      "Epoch 60 batch 100 loss 0.083368 acc 0.843750\n",
      "Epoch 60 batch 200 loss 0.093842 acc 0.812500\n",
      "Epoch 60 batch 300 loss 0.084680 acc 0.875000\n",
      "Epoch 60 batch 400 loss 0.103265 acc 0.812500\n",
      "Epoch 60 batch 500 loss 0.073247 acc 0.906250\n",
      "Epoch 60 batch 600 loss 0.099753 acc 0.843750\n",
      "Epoch 60 batch 700 loss 0.089047 acc 0.843750\n",
      "Epoch 60 batch 800 loss 0.079554 acc 0.875000\n",
      "Epoch 60 batch 900 loss 0.072616 acc 0.875000\n",
      "Epoch 60 batch 1000 loss 0.079659 acc 0.843750\n",
      "Epoch 60 batch 1100 loss 0.099525 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 61 batch 0 loss 0.041286 acc 0.937500\n",
      "Epoch 61 batch 100 loss 0.083596 acc 0.843750\n",
      "Epoch 61 batch 200 loss 0.093988 acc 0.812500\n",
      "Epoch 61 batch 300 loss 0.087107 acc 0.875000\n",
      "Epoch 61 batch 400 loss 0.103433 acc 0.812500\n",
      "Epoch 61 batch 500 loss 0.073390 acc 0.906250\n",
      "Epoch 61 batch 600 loss 0.099816 acc 0.843750\n",
      "Epoch 61 batch 700 loss 0.088382 acc 0.843750\n",
      "Epoch 61 batch 800 loss 0.079446 acc 0.875000\n",
      "Epoch 61 batch 900 loss 0.072640 acc 0.875000\n",
      "Epoch 61 batch 1000 loss 0.079507 acc 0.843750\n",
      "Epoch 61 batch 1100 loss 0.099886 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 62 batch 0 loss 0.043777 acc 0.937500\n",
      "Epoch 62 batch 100 loss 0.083541 acc 0.843750\n",
      "Epoch 62 batch 200 loss 0.094011 acc 0.812500\n",
      "Epoch 62 batch 300 loss 0.085397 acc 0.875000\n",
      "Epoch 62 batch 400 loss 0.103445 acc 0.812500\n",
      "Epoch 62 batch 500 loss 0.073429 acc 0.906250\n",
      "Epoch 62 batch 600 loss 0.099750 acc 0.843750\n",
      "Epoch 62 batch 700 loss 0.087665 acc 0.843750\n",
      "Epoch 62 batch 800 loss 0.079406 acc 0.875000\n",
      "Epoch 62 batch 900 loss 0.072606 acc 0.875000\n",
      "Epoch 62 batch 1000 loss 0.079408 acc 0.843750\n",
      "Epoch 62 batch 1100 loss 0.099456 acc 0.781250\n",
      "Validation accuracy: 0.829189\n",
      "Epoch 63 batch 0 loss 0.041099 acc 0.937500\n",
      "Epoch 63 batch 100 loss 0.083664 acc 0.843750\n",
      "Epoch 63 batch 200 loss 0.094018 acc 0.812500\n",
      "Epoch 63 batch 300 loss 0.086383 acc 0.875000\n",
      "Epoch 63 batch 400 loss 0.103379 acc 0.812500\n",
      "Epoch 63 batch 500 loss 0.073756 acc 0.906250\n",
      "Epoch 63 batch 600 loss 0.099822 acc 0.843750\n",
      "Epoch 63 batch 700 loss 0.088319 acc 0.843750\n",
      "Epoch 63 batch 800 loss 0.079278 acc 0.875000\n",
      "Epoch 63 batch 900 loss 0.072715 acc 0.875000\n",
      "Epoch 63 batch 1000 loss 0.079499 acc 0.843750\n",
      "Epoch 63 batch 1100 loss 0.100022 acc 0.781250\n",
      "Validation accuracy: 0.829584\n",
      "Epoch 64 batch 0 loss 0.042795 acc 0.937500\n",
      "Epoch 64 batch 100 loss 0.083660 acc 0.843750\n",
      "Epoch 64 batch 200 loss 0.094066 acc 0.812500\n",
      "Epoch 64 batch 300 loss 0.086145 acc 0.875000\n",
      "Epoch 64 batch 400 loss 0.103331 acc 0.812500\n",
      "Epoch 64 batch 500 loss 0.073909 acc 0.906250\n",
      "Epoch 64 batch 600 loss 0.099799 acc 0.843750\n",
      "Epoch 64 batch 700 loss 0.088285 acc 0.843750\n",
      "Epoch 64 batch 800 loss 0.079215 acc 0.875000\n",
      "Epoch 64 batch 900 loss 0.072773 acc 0.875000\n",
      "Epoch 64 batch 1000 loss 0.079502 acc 0.843750\n",
      "Epoch 64 batch 1100 loss 0.100815 acc 0.781250\n",
      "Validation accuracy: 0.829663\n",
      "Epoch 65 batch 0 loss 0.042119 acc 0.937500\n",
      "Epoch 65 batch 100 loss 0.083601 acc 0.843750\n",
      "Epoch 65 batch 200 loss 0.094070 acc 0.812500\n",
      "Epoch 65 batch 300 loss 0.081152 acc 0.875000\n",
      "Epoch 65 batch 400 loss 0.103572 acc 0.812500\n",
      "Epoch 65 batch 500 loss 0.074147 acc 0.906250\n",
      "Epoch 65 batch 600 loss 0.099670 acc 0.843750\n",
      "Epoch 65 batch 700 loss 0.088404 acc 0.843750\n",
      "Epoch 65 batch 800 loss 0.079189 acc 0.875000\n",
      "Epoch 65 batch 900 loss 0.072797 acc 0.875000\n",
      "Epoch 65 batch 1000 loss 0.079479 acc 0.843750\n",
      "Epoch 65 batch 1100 loss 0.098713 acc 0.781250\n",
      "Validation accuracy: 0.829505\n",
      "Epoch 66 batch 0 loss 0.041104 acc 0.937500\n",
      "Epoch 66 batch 100 loss 0.083789 acc 0.843750\n",
      "Epoch 66 batch 200 loss 0.094051 acc 0.812500\n",
      "Epoch 66 batch 300 loss 0.085137 acc 0.875000\n",
      "Epoch 66 batch 400 loss 0.103119 acc 0.812500\n",
      "Epoch 66 batch 500 loss 0.074104 acc 0.906250\n",
      "Epoch 66 batch 600 loss 0.099756 acc 0.843750\n",
      "Epoch 66 batch 700 loss 0.088056 acc 0.843750\n",
      "Epoch 66 batch 800 loss 0.079222 acc 0.875000\n",
      "Epoch 66 batch 900 loss 0.072772 acc 0.875000\n",
      "Epoch 66 batch 1000 loss 0.079543 acc 0.843750\n",
      "Epoch 66 batch 1100 loss 0.099072 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 67 batch 0 loss 0.041134 acc 0.937500\n",
      "Epoch 67 batch 100 loss 0.083912 acc 0.843750\n",
      "Epoch 67 batch 200 loss 0.094120 acc 0.812500\n",
      "Epoch 67 batch 300 loss 0.081259 acc 0.875000\n",
      "Epoch 67 batch 400 loss 0.103151 acc 0.812500\n",
      "Epoch 67 batch 500 loss 0.074343 acc 0.906250\n",
      "Epoch 67 batch 600 loss 0.099632 acc 0.843750\n",
      "Epoch 67 batch 700 loss 0.087561 acc 0.843750\n",
      "Epoch 67 batch 800 loss 0.079369 acc 0.875000\n",
      "Epoch 67 batch 900 loss 0.072649 acc 0.875000\n",
      "Epoch 67 batch 1000 loss 0.079598 acc 0.843750\n",
      "Epoch 67 batch 1100 loss 0.099401 acc 0.781250\n",
      "Validation accuracy: 0.829347\n",
      "Epoch 68 batch 0 loss 0.041504 acc 0.937500\n",
      "Epoch 68 batch 100 loss 0.083927 acc 0.843750\n",
      "Epoch 68 batch 200 loss 0.094143 acc 0.812500\n",
      "Epoch 68 batch 300 loss 0.082813 acc 0.875000\n",
      "Epoch 68 batch 400 loss 0.103244 acc 0.812500\n",
      "Epoch 68 batch 500 loss 0.074486 acc 0.906250\n",
      "Epoch 68 batch 600 loss 0.099377 acc 0.843750\n",
      "Epoch 68 batch 700 loss 0.087017 acc 0.843750\n",
      "Epoch 68 batch 800 loss 0.079539 acc 0.875000\n",
      "Epoch 68 batch 900 loss 0.072536 acc 0.875000\n",
      "Epoch 68 batch 1000 loss 0.079702 acc 0.843750\n",
      "Epoch 68 batch 1100 loss 0.099857 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 69 batch 0 loss 0.041942 acc 0.937500\n",
      "Epoch 69 batch 100 loss 0.083839 acc 0.843750\n",
      "Epoch 69 batch 200 loss 0.094086 acc 0.812500\n",
      "Epoch 69 batch 300 loss 0.083668 acc 0.875000\n",
      "Epoch 69 batch 400 loss 0.103120 acc 0.812500\n",
      "Epoch 69 batch 500 loss 0.074532 acc 0.906250\n",
      "Epoch 69 batch 600 loss 0.099416 acc 0.843750\n",
      "Epoch 69 batch 700 loss 0.086853 acc 0.843750\n",
      "Epoch 69 batch 800 loss 0.079662 acc 0.875000\n",
      "Epoch 69 batch 900 loss 0.072457 acc 0.875000\n",
      "Epoch 69 batch 1000 loss 0.079763 acc 0.843750\n",
      "Epoch 69 batch 1100 loss 0.099587 acc 0.781250\n",
      "Validation accuracy: 0.829584\n",
      "Epoch 70 batch 0 loss 0.041551 acc 0.937500\n",
      "Epoch 70 batch 100 loss 0.084018 acc 0.843750\n",
      "Epoch 70 batch 200 loss 0.094113 acc 0.812500\n",
      "Epoch 70 batch 300 loss 0.083072 acc 0.875000\n",
      "Epoch 70 batch 400 loss 0.103022 acc 0.812500\n",
      "Epoch 70 batch 500 loss 0.074825 acc 0.906250\n",
      "Epoch 70 batch 600 loss 0.099160 acc 0.843750\n",
      "Epoch 70 batch 700 loss 0.086794 acc 0.843750\n",
      "Epoch 70 batch 800 loss 0.079664 acc 0.875000\n",
      "Epoch 70 batch 900 loss 0.072387 acc 0.875000\n",
      "Epoch 70 batch 1000 loss 0.079881 acc 0.843750\n",
      "Epoch 70 batch 1100 loss 0.100105 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 71 batch 0 loss 0.042364 acc 0.937500\n",
      "Epoch 71 batch 100 loss 0.083759 acc 0.843750\n",
      "Epoch 71 batch 200 loss 0.094023 acc 0.812500\n",
      "Epoch 71 batch 300 loss 0.082817 acc 0.875000\n",
      "Epoch 71 batch 400 loss 0.103106 acc 0.812500\n",
      "Epoch 71 batch 500 loss 0.074919 acc 0.906250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71 batch 600 loss 0.099178 acc 0.843750\n",
      "Epoch 71 batch 700 loss 0.086844 acc 0.843750\n",
      "Epoch 71 batch 800 loss 0.079654 acc 0.875000\n",
      "Epoch 71 batch 900 loss 0.072266 acc 0.875000\n",
      "Epoch 71 batch 1000 loss 0.080006 acc 0.843750\n",
      "Epoch 71 batch 1100 loss 0.102016 acc 0.781250\n",
      "Validation accuracy: 0.830215\n",
      "Epoch 72 batch 0 loss 0.041463 acc 0.937500\n",
      "Epoch 72 batch 100 loss 0.084061 acc 0.843750\n",
      "Epoch 72 batch 200 loss 0.094004 acc 0.812500\n",
      "Epoch 72 batch 300 loss 0.081708 acc 0.875000\n",
      "Epoch 72 batch 400 loss 0.103288 acc 0.812500\n",
      "Epoch 72 batch 500 loss 0.075190 acc 0.906250\n",
      "Epoch 72 batch 600 loss 0.098694 acc 0.843750\n",
      "Epoch 72 batch 700 loss 0.086881 acc 0.843750\n",
      "Epoch 72 batch 800 loss 0.079715 acc 0.875000\n",
      "Epoch 72 batch 900 loss 0.072132 acc 0.875000\n",
      "Epoch 72 batch 1000 loss 0.080054 acc 0.843750\n",
      "Epoch 72 batch 1100 loss 0.100082 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 73 batch 0 loss 0.042178 acc 0.937500\n",
      "Epoch 73 batch 100 loss 0.084203 acc 0.843750\n",
      "Epoch 73 batch 200 loss 0.094096 acc 0.812500\n",
      "Epoch 73 batch 300 loss 0.083167 acc 0.875000\n",
      "Epoch 73 batch 400 loss 0.102979 acc 0.812500\n",
      "Epoch 73 batch 500 loss 0.075364 acc 0.906250\n",
      "Epoch 73 batch 600 loss 0.098653 acc 0.843750\n",
      "Epoch 73 batch 700 loss 0.086952 acc 0.843750\n",
      "Epoch 73 batch 800 loss 0.079919 acc 0.875000\n",
      "Epoch 73 batch 900 loss 0.072186 acc 0.875000\n",
      "Epoch 73 batch 1000 loss 0.080010 acc 0.843750\n",
      "Epoch 73 batch 1100 loss 0.103982 acc 0.781250\n",
      "Validation accuracy: 0.829742\n",
      "Epoch 74 batch 0 loss 0.041634 acc 0.937500\n",
      "Epoch 74 batch 100 loss 0.084063 acc 0.843750\n",
      "Epoch 74 batch 200 loss 0.094055 acc 0.812500\n",
      "Epoch 74 batch 300 loss 0.086438 acc 0.875000\n",
      "Epoch 74 batch 400 loss 0.102957 acc 0.812500\n",
      "Epoch 74 batch 500 loss 0.075716 acc 0.906250\n",
      "Epoch 74 batch 600 loss 0.098211 acc 0.843750\n",
      "Epoch 74 batch 700 loss 0.087067 acc 0.843750\n",
      "Epoch 74 batch 800 loss 0.079788 acc 0.875000\n",
      "Epoch 74 batch 900 loss 0.072012 acc 0.875000\n",
      "Epoch 74 batch 1000 loss 0.080153 acc 0.843750\n",
      "Epoch 74 batch 1100 loss 0.100350 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 75 batch 0 loss 0.041782 acc 0.937500\n",
      "Epoch 75 batch 100 loss 0.084452 acc 0.843750\n",
      "Epoch 75 batch 200 loss 0.094122 acc 0.812500\n",
      "Epoch 75 batch 300 loss 0.081481 acc 0.875000\n",
      "Epoch 75 batch 400 loss 0.103224 acc 0.812500\n",
      "Epoch 75 batch 500 loss 0.074595 acc 0.906250\n",
      "Epoch 75 batch 600 loss 0.098449 acc 0.843750\n",
      "Epoch 75 batch 700 loss 0.087015 acc 0.843750\n",
      "Epoch 75 batch 800 loss 0.079850 acc 0.875000\n",
      "Epoch 75 batch 900 loss 0.071985 acc 0.875000\n",
      "Epoch 75 batch 1000 loss 0.080288 acc 0.843750\n",
      "Epoch 75 batch 1100 loss 0.101598 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 76 batch 0 loss 0.041954 acc 0.937500\n",
      "Epoch 76 batch 100 loss 0.084202 acc 0.843750\n",
      "Epoch 76 batch 200 loss 0.094088 acc 0.812500\n",
      "Epoch 76 batch 300 loss 0.083458 acc 0.875000\n",
      "Epoch 76 batch 400 loss 0.103074 acc 0.812500\n",
      "Epoch 76 batch 500 loss 0.075642 acc 0.906250\n",
      "Epoch 76 batch 600 loss 0.097960 acc 0.843750\n",
      "Epoch 76 batch 700 loss 0.087262 acc 0.843750\n",
      "Epoch 76 batch 800 loss 0.079767 acc 0.875000\n",
      "Epoch 76 batch 900 loss 0.071994 acc 0.875000\n",
      "Epoch 76 batch 1000 loss 0.080256 acc 0.843750\n",
      "Epoch 76 batch 1100 loss 0.101073 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 77 batch 0 loss 0.041450 acc 0.937500\n",
      "Epoch 77 batch 100 loss 0.084349 acc 0.843750\n",
      "Epoch 77 batch 200 loss 0.094098 acc 0.812500\n",
      "Epoch 77 batch 300 loss 0.086654 acc 0.875000\n",
      "Epoch 77 batch 400 loss 0.103159 acc 0.812500\n",
      "Epoch 77 batch 500 loss 0.075138 acc 0.906250\n",
      "Epoch 77 batch 600 loss 0.097696 acc 0.843750\n",
      "Epoch 77 batch 700 loss 0.087410 acc 0.843750\n",
      "Epoch 77 batch 800 loss 0.079629 acc 0.875000\n",
      "Epoch 77 batch 900 loss 0.071995 acc 0.875000\n",
      "Epoch 77 batch 1000 loss 0.080321 acc 0.843750\n",
      "Epoch 77 batch 1100 loss 0.101598 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 78 batch 0 loss 0.043234 acc 0.937500\n",
      "Epoch 78 batch 100 loss 0.084343 acc 0.843750\n",
      "Epoch 78 batch 200 loss 0.094147 acc 0.812500\n",
      "Epoch 78 batch 300 loss 0.082121 acc 0.875000\n",
      "Epoch 78 batch 400 loss 0.102890 acc 0.812500\n",
      "Epoch 78 batch 500 loss 0.075967 acc 0.906250\n",
      "Epoch 78 batch 600 loss 0.097381 acc 0.843750\n",
      "Epoch 78 batch 700 loss 0.087490 acc 0.843750\n",
      "Epoch 78 batch 800 loss 0.079726 acc 0.875000\n",
      "Epoch 78 batch 900 loss 0.072040 acc 0.875000\n",
      "Epoch 78 batch 1000 loss 0.080346 acc 0.843750\n",
      "Epoch 78 batch 1100 loss 0.101459 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 79 batch 0 loss 0.044061 acc 0.937500\n",
      "Epoch 79 batch 100 loss 0.084261 acc 0.843750\n",
      "Epoch 79 batch 200 loss 0.094164 acc 0.812500\n",
      "Epoch 79 batch 300 loss 0.083390 acc 0.875000\n",
      "Epoch 79 batch 400 loss 0.102833 acc 0.812500\n",
      "Epoch 79 batch 500 loss 0.075969 acc 0.906250\n",
      "Epoch 79 batch 600 loss 0.097243 acc 0.843750\n",
      "Epoch 79 batch 700 loss 0.087550 acc 0.843750\n",
      "Epoch 79 batch 800 loss 0.079979 acc 0.875000\n",
      "Epoch 79 batch 900 loss 0.072010 acc 0.875000\n",
      "Epoch 79 batch 1000 loss 0.080379 acc 0.843750\n",
      "Epoch 79 batch 1100 loss 0.101146 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 80 batch 0 loss 0.041656 acc 0.937500\n",
      "Epoch 80 batch 100 loss 0.084581 acc 0.843750\n",
      "Epoch 80 batch 200 loss 0.094179 acc 0.812500\n",
      "Epoch 80 batch 300 loss 0.085864 acc 0.875000\n",
      "Epoch 80 batch 400 loss 0.102715 acc 0.812500\n",
      "Epoch 80 batch 500 loss 0.075689 acc 0.906250\n",
      "Epoch 80 batch 600 loss 0.097215 acc 0.843750\n",
      "Epoch 80 batch 700 loss 0.087638 acc 0.843750\n",
      "Epoch 80 batch 800 loss 0.079545 acc 0.875000\n",
      "Epoch 80 batch 900 loss 0.071984 acc 0.875000\n",
      "Epoch 80 batch 1000 loss 0.080376 acc 0.843750\n",
      "Epoch 80 batch 1100 loss 0.101972 acc 0.781250\n",
      "Validation accuracy: 0.830215\n",
      "Epoch 81 batch 0 loss 0.042071 acc 0.937500\n",
      "Epoch 81 batch 100 loss 0.084252 acc 0.843750\n",
      "Epoch 81 batch 200 loss 0.094132 acc 0.812500\n",
      "Epoch 81 batch 300 loss 0.081683 acc 0.875000\n",
      "Epoch 81 batch 400 loss 0.103084 acc 0.812500\n",
      "Epoch 81 batch 500 loss 0.076076 acc 0.906250\n",
      "Epoch 81 batch 600 loss 0.097056 acc 0.843750\n",
      "Epoch 81 batch 700 loss 0.087700 acc 0.843750\n",
      "Epoch 81 batch 800 loss 0.079602 acc 0.875000\n",
      "Epoch 81 batch 900 loss 0.071955 acc 0.875000\n",
      "Epoch 81 batch 1000 loss 0.080365 acc 0.843750\n",
      "Epoch 81 batch 1100 loss 0.101736 acc 0.781250\n",
      "Validation accuracy: 0.832662\n",
      "Epoch 82 batch 0 loss 0.045541 acc 0.937500\n",
      "Epoch 82 batch 100 loss 0.084430 acc 0.843750\n",
      "Epoch 82 batch 200 loss 0.094219 acc 0.812500\n",
      "Epoch 82 batch 300 loss 0.079495 acc 0.875000\n",
      "Epoch 82 batch 400 loss 0.102975 acc 0.812500\n",
      "Epoch 82 batch 500 loss 0.076251 acc 0.906250\n",
      "Epoch 82 batch 600 loss 0.096902 acc 0.843750\n",
      "Epoch 82 batch 700 loss 0.087802 acc 0.843750\n",
      "Epoch 82 batch 800 loss 0.079580 acc 0.875000\n",
      "Epoch 82 batch 900 loss 0.071963 acc 0.875000\n",
      "Epoch 82 batch 1000 loss 0.080306 acc 0.843750\n",
      "Epoch 82 batch 1100 loss 0.106016 acc 0.781250\n",
      "Validation accuracy: 0.830926\n",
      "Epoch 83 batch 0 loss 0.042579 acc 0.937500\n",
      "Epoch 83 batch 100 loss 0.084385 acc 0.843750\n",
      "Epoch 83 batch 200 loss 0.094146 acc 0.812500\n",
      "Epoch 83 batch 300 loss 0.084283 acc 0.875000\n",
      "Epoch 83 batch 400 loss 0.102653 acc 0.812500\n",
      "Epoch 83 batch 500 loss 0.075504 acc 0.906250\n",
      "Epoch 83 batch 600 loss 0.097006 acc 0.843750\n",
      "Epoch 83 batch 700 loss 0.087768 acc 0.843750\n",
      "Epoch 83 batch 800 loss 0.079474 acc 0.875000\n",
      "Epoch 83 batch 900 loss 0.071969 acc 0.875000\n",
      "Epoch 83 batch 1000 loss 0.080379 acc 0.843750\n",
      "Epoch 83 batch 1100 loss 0.102173 acc 0.781250\n",
      "Validation accuracy: 0.830294\n",
      "Epoch 84 batch 0 loss 0.041995 acc 0.937500\n",
      "Epoch 84 batch 100 loss 0.084517 acc 0.843750\n",
      "Epoch 84 batch 200 loss 0.094193 acc 0.812500\n",
      "Epoch 84 batch 300 loss 0.084058 acc 0.875000\n",
      "Epoch 84 batch 400 loss 0.102798 acc 0.812500\n",
      "Epoch 84 batch 500 loss 0.076211 acc 0.906250\n",
      "Epoch 84 batch 600 loss 0.096714 acc 0.843750\n",
      "Epoch 84 batch 700 loss 0.087790 acc 0.843750\n",
      "Epoch 84 batch 800 loss 0.079434 acc 0.875000\n",
      "Epoch 84 batch 900 loss 0.071946 acc 0.875000\n",
      "Epoch 84 batch 1000 loss 0.080396 acc 0.843750\n",
      "Epoch 84 batch 1100 loss 0.101797 acc 0.781250\n",
      "Validation accuracy: 0.830531\n",
      "Epoch 85 batch 0 loss 0.042119 acc 0.937500\n",
      "Epoch 85 batch 100 loss 0.084801 acc 0.843750\n",
      "Epoch 85 batch 200 loss 0.094242 acc 0.812500\n",
      "Epoch 85 batch 300 loss 0.086173 acc 0.875000\n",
      "Epoch 85 batch 400 loss 0.102726 acc 0.812500\n",
      "Epoch 85 batch 500 loss 0.075469 acc 0.906250\n",
      "Epoch 85 batch 600 loss 0.096995 acc 0.843750\n",
      "Epoch 85 batch 700 loss 0.087691 acc 0.843750\n",
      "Epoch 85 batch 800 loss 0.079405 acc 0.875000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85 batch 900 loss 0.071979 acc 0.875000\n",
      "Epoch 85 batch 1000 loss 0.080403 acc 0.843750\n",
      "Epoch 85 batch 1100 loss 0.104470 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 86 batch 0 loss 0.041771 acc 0.937500\n",
      "Epoch 86 batch 100 loss 0.084426 acc 0.843750\n",
      "Epoch 86 batch 200 loss 0.094202 acc 0.812500\n",
      "Epoch 86 batch 300 loss 0.085865 acc 0.875000\n",
      "Epoch 86 batch 400 loss 0.102713 acc 0.812500\n",
      "Epoch 86 batch 500 loss 0.076175 acc 0.906250\n",
      "Epoch 86 batch 600 loss 0.096601 acc 0.843750\n",
      "Epoch 86 batch 700 loss 0.087806 acc 0.843750\n",
      "Epoch 86 batch 800 loss 0.079369 acc 0.875000\n",
      "Epoch 86 batch 900 loss 0.071939 acc 0.875000\n",
      "Epoch 86 batch 1000 loss 0.080399 acc 0.843750\n",
      "Epoch 86 batch 1100 loss 0.102185 acc 0.781250\n",
      "Validation accuracy: 0.832426\n",
      "Epoch 87 batch 0 loss 0.045230 acc 0.937500\n",
      "Epoch 87 batch 100 loss 0.084393 acc 0.843750\n",
      "Epoch 87 batch 200 loss 0.094265 acc 0.812500\n",
      "Epoch 87 batch 300 loss 0.083560 acc 0.875000\n",
      "Epoch 87 batch 400 loss 0.102853 acc 0.812500\n",
      "Epoch 87 batch 500 loss 0.076405 acc 0.906250\n",
      "Epoch 87 batch 600 loss 0.096507 acc 0.843750\n",
      "Epoch 87 batch 700 loss 0.088076 acc 0.843750\n",
      "Epoch 87 batch 800 loss 0.079465 acc 0.875000\n",
      "Epoch 87 batch 900 loss 0.071993 acc 0.875000\n",
      "Epoch 87 batch 1000 loss 0.080440 acc 0.843750\n",
      "Epoch 87 batch 1100 loss 0.102004 acc 0.781250\n",
      "Validation accuracy: 0.830926\n",
      "Epoch 88 batch 0 loss 0.042381 acc 0.937500\n",
      "Epoch 88 batch 100 loss 0.084600 acc 0.843750\n",
      "Epoch 88 batch 200 loss 0.094274 acc 0.812500\n",
      "Epoch 88 batch 300 loss 0.086190 acc 0.875000\n",
      "Epoch 88 batch 400 loss 0.102716 acc 0.812500\n",
      "Epoch 88 batch 500 loss 0.075244 acc 0.906250\n",
      "Epoch 88 batch 600 loss 0.096776 acc 0.843750\n",
      "Epoch 88 batch 700 loss 0.088010 acc 0.843750\n",
      "Epoch 88 batch 800 loss 0.079216 acc 0.875000\n",
      "Epoch 88 batch 900 loss 0.071961 acc 0.875000\n",
      "Epoch 88 batch 1000 loss 0.080438 acc 0.843750\n",
      "Epoch 88 batch 1100 loss 0.103166 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 89 batch 0 loss 0.041237 acc 0.937500\n",
      "Epoch 89 batch 100 loss 0.084496 acc 0.843750\n",
      "Epoch 89 batch 200 loss 0.094197 acc 0.812500\n",
      "Epoch 89 batch 300 loss 0.086109 acc 0.875000\n",
      "Epoch 89 batch 400 loss 0.102780 acc 0.812500\n",
      "Epoch 89 batch 500 loss 0.075661 acc 0.906250\n",
      "Epoch 89 batch 600 loss 0.096718 acc 0.843750\n",
      "Epoch 89 batch 700 loss 0.087877 acc 0.843750\n",
      "Epoch 89 batch 800 loss 0.079248 acc 0.875000\n",
      "Epoch 89 batch 900 loss 0.071951 acc 0.875000\n",
      "Epoch 89 batch 1000 loss 0.080479 acc 0.843750\n",
      "Epoch 89 batch 1100 loss 0.102228 acc 0.781250\n",
      "Validation accuracy: 0.829663\n",
      "Epoch 90 batch 0 loss 0.041004 acc 0.937500\n",
      "Epoch 90 batch 100 loss 0.084834 acc 0.843750\n",
      "Epoch 90 batch 200 loss 0.094283 acc 0.812500\n",
      "Epoch 90 batch 300 loss 0.081494 acc 0.875000\n",
      "Epoch 90 batch 400 loss 0.102943 acc 0.812500\n",
      "Epoch 90 batch 500 loss 0.075817 acc 0.906250\n",
      "Epoch 90 batch 600 loss 0.096582 acc 0.843750\n",
      "Epoch 90 batch 700 loss 0.087894 acc 0.843750\n",
      "Epoch 90 batch 800 loss 0.079228 acc 0.875000\n",
      "Epoch 90 batch 900 loss 0.071943 acc 0.875000\n",
      "Epoch 90 batch 1000 loss 0.080476 acc 0.843750\n",
      "Epoch 90 batch 1100 loss 0.103840 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 91 batch 0 loss 0.040183 acc 0.937500\n",
      "Epoch 91 batch 100 loss 0.084790 acc 0.843750\n",
      "Epoch 91 batch 200 loss 0.094233 acc 0.812500\n",
      "Epoch 91 batch 300 loss 0.085272 acc 0.875000\n",
      "Epoch 91 batch 400 loss 0.102849 acc 0.812500\n",
      "Epoch 91 batch 500 loss 0.075269 acc 0.906250\n",
      "Epoch 91 batch 600 loss 0.096700 acc 0.843750\n",
      "Epoch 91 batch 700 loss 0.087959 acc 0.843750\n",
      "Epoch 91 batch 800 loss 0.079192 acc 0.875000\n",
      "Epoch 91 batch 900 loss 0.071956 acc 0.875000\n",
      "Epoch 91 batch 1000 loss 0.080460 acc 0.843750\n",
      "Epoch 91 batch 1100 loss 0.102575 acc 0.781250\n",
      "Validation accuracy: 0.831321\n",
      "Epoch 92 batch 0 loss 0.042436 acc 0.937500\n",
      "Epoch 92 batch 100 loss 0.084816 acc 0.843750\n",
      "Epoch 92 batch 200 loss 0.094331 acc 0.812500\n",
      "Epoch 92 batch 300 loss 0.085474 acc 0.875000\n",
      "Epoch 92 batch 400 loss 0.102705 acc 0.812500\n",
      "Epoch 92 batch 500 loss 0.076046 acc 0.906250\n",
      "Epoch 92 batch 600 loss 0.096414 acc 0.843750\n",
      "Epoch 92 batch 700 loss 0.088002 acc 0.843750\n",
      "Epoch 92 batch 800 loss 0.079120 acc 0.875000\n",
      "Epoch 92 batch 900 loss 0.071960 acc 0.875000\n",
      "Epoch 92 batch 1000 loss 0.080440 acc 0.843750\n",
      "Epoch 92 batch 1100 loss 0.103549 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 93 batch 0 loss 0.042797 acc 0.937500\n",
      "Epoch 93 batch 100 loss 0.084743 acc 0.843750\n",
      "Epoch 93 batch 200 loss 0.094307 acc 0.812500\n",
      "Epoch 93 batch 300 loss 0.081973 acc 0.875000\n",
      "Epoch 93 batch 400 loss 0.102882 acc 0.812500\n",
      "Epoch 93 batch 500 loss 0.076143 acc 0.906250\n",
      "Epoch 93 batch 600 loss 0.096372 acc 0.843750\n",
      "Epoch 93 batch 700 loss 0.088000 acc 0.843750\n",
      "Epoch 93 batch 800 loss 0.079169 acc 0.875000\n",
      "Epoch 93 batch 900 loss 0.071929 acc 0.875000\n",
      "Epoch 93 batch 1000 loss 0.080481 acc 0.843750\n",
      "Epoch 93 batch 1100 loss 0.102597 acc 0.781250\n",
      "Validation accuracy: 0.831005\n",
      "Epoch 94 batch 0 loss 0.042171 acc 0.937500\n",
      "Epoch 94 batch 100 loss 0.084810 acc 0.843750\n",
      "Epoch 94 batch 200 loss 0.094332 acc 0.812500\n",
      "Epoch 94 batch 300 loss 0.083274 acc 0.875000\n",
      "Epoch 94 batch 400 loss 0.102594 acc 0.812500\n",
      "Epoch 94 batch 500 loss 0.075129 acc 0.906250\n",
      "Epoch 94 batch 600 loss 0.096640 acc 0.843750\n",
      "Epoch 94 batch 700 loss 0.087980 acc 0.843750\n",
      "Epoch 94 batch 800 loss 0.079070 acc 0.875000\n",
      "Epoch 94 batch 900 loss 0.071949 acc 0.875000\n",
      "Epoch 94 batch 1000 loss 0.080516 acc 0.843750\n",
      "Epoch 94 batch 1100 loss 0.102647 acc 0.781250\n",
      "Validation accuracy: 0.830294\n",
      "Epoch 95 batch 0 loss 0.041639 acc 0.937500\n",
      "Epoch 95 batch 100 loss 0.084703 acc 0.843750\n",
      "Epoch 95 batch 200 loss 0.094335 acc 0.812500\n",
      "Epoch 95 batch 300 loss 0.085261 acc 0.875000\n",
      "Epoch 95 batch 400 loss 0.102720 acc 0.812500\n",
      "Epoch 95 batch 500 loss 0.076337 acc 0.906250\n",
      "Epoch 95 batch 600 loss 0.096210 acc 0.843750\n",
      "Epoch 95 batch 700 loss 0.088098 acc 0.843750\n",
      "Epoch 95 batch 800 loss 0.079065 acc 0.875000\n",
      "Epoch 95 batch 900 loss 0.071943 acc 0.875000\n",
      "Epoch 95 batch 1000 loss 0.080447 acc 0.843750\n",
      "Epoch 95 batch 1100 loss 0.102446 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 96 batch 0 loss 0.040881 acc 0.937500\n",
      "Epoch 96 batch 100 loss 0.084869 acc 0.843750\n",
      "Epoch 96 batch 200 loss 0.094359 acc 0.812500\n",
      "Epoch 96 batch 300 loss 0.081464 acc 0.875000\n",
      "Epoch 96 batch 400 loss 0.102988 acc 0.812500\n",
      "Epoch 96 batch 500 loss 0.075572 acc 0.906250\n",
      "Epoch 96 batch 600 loss 0.096518 acc 0.843750\n",
      "Epoch 96 batch 700 loss 0.087950 acc 0.843750\n",
      "Epoch 96 batch 800 loss 0.079016 acc 0.875000\n",
      "Epoch 96 batch 900 loss 0.071966 acc 0.875000\n",
      "Epoch 96 batch 1000 loss 0.080419 acc 0.843750\n",
      "Epoch 96 batch 1100 loss 0.103915 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 97 batch 0 loss 0.040305 acc 0.937500\n",
      "Epoch 97 batch 100 loss 0.085135 acc 0.843750\n",
      "Epoch 97 batch 200 loss 0.094327 acc 0.812500\n",
      "Epoch 97 batch 300 loss 0.082837 acc 0.875000\n",
      "Epoch 97 batch 400 loss 0.103021 acc 0.812500\n",
      "Epoch 97 batch 500 loss 0.075241 acc 0.906250\n",
      "Epoch 97 batch 600 loss 0.096356 acc 0.843750\n",
      "Epoch 97 batch 700 loss 0.088057 acc 0.843750\n",
      "Epoch 97 batch 800 loss 0.079017 acc 0.875000\n",
      "Epoch 97 batch 900 loss 0.071957 acc 0.875000\n",
      "Epoch 97 batch 1000 loss 0.080504 acc 0.843750\n",
      "Epoch 97 batch 1100 loss 0.104648 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 98 batch 0 loss 0.040900 acc 0.937500\n",
      "Epoch 98 batch 100 loss 0.084697 acc 0.843750\n",
      "Epoch 98 batch 200 loss 0.094321 acc 0.812500\n",
      "Epoch 98 batch 300 loss 0.081469 acc 0.875000\n",
      "Epoch 98 batch 400 loss 0.102777 acc 0.812500\n",
      "Epoch 98 batch 500 loss 0.075733 acc 0.906250\n",
      "Epoch 98 batch 600 loss 0.096336 acc 0.843750\n",
      "Epoch 98 batch 700 loss 0.088024 acc 0.843750\n",
      "Epoch 98 batch 800 loss 0.079034 acc 0.875000\n",
      "Epoch 98 batch 900 loss 0.071934 acc 0.875000\n",
      "Epoch 98 batch 1000 loss 0.080469 acc 0.843750\n",
      "Epoch 98 batch 1100 loss 0.104357 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 99 batch 0 loss 0.041135 acc 0.937500\n",
      "Epoch 99 batch 100 loss 0.084593 acc 0.843750\n",
      "Epoch 99 batch 200 loss 0.094408 acc 0.812500\n",
      "Epoch 99 batch 300 loss 0.081918 acc 0.875000\n",
      "Epoch 99 batch 400 loss 0.104380 acc 0.812500\n",
      "Epoch 99 batch 500 loss 0.075542 acc 0.906250\n",
      "Epoch 99 batch 600 loss 0.096428 acc 0.843750\n",
      "Epoch 99 batch 700 loss 0.088058 acc 0.843750\n",
      "Epoch 99 batch 800 loss 0.078978 acc 0.875000\n",
      "Epoch 99 batch 900 loss 0.071941 acc 0.875000\n",
      "Epoch 99 batch 1000 loss 0.080527 acc 0.843750\n",
      "Epoch 99 batch 1100 loss 0.103793 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.830689\n",
      "Epoch 100 batch 0 loss 0.042047 acc 0.937500\n",
      "Epoch 100 batch 100 loss 0.084805 acc 0.843750\n",
      "Epoch 100 batch 200 loss 0.094387 acc 0.812500\n",
      "Epoch 100 batch 300 loss 0.082038 acc 0.875000\n",
      "Epoch 100 batch 400 loss 0.102912 acc 0.812500\n",
      "Epoch 100 batch 500 loss 0.075694 acc 0.906250\n",
      "Epoch 100 batch 600 loss 0.096277 acc 0.843750\n",
      "Epoch 100 batch 700 loss 0.088155 acc 0.843750\n",
      "Epoch 100 batch 800 loss 0.078920 acc 0.875000\n",
      "Epoch 100 batch 900 loss 0.071946 acc 0.875000\n",
      "Epoch 100 batch 1000 loss 0.080469 acc 0.843750\n",
      "Epoch 100 batch 1100 loss 0.104749 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 101 batch 0 loss 0.043937 acc 0.937500\n",
      "Epoch 101 batch 100 loss 0.085013 acc 0.843750\n",
      "Epoch 101 batch 200 loss 0.094407 acc 0.812500\n",
      "Epoch 101 batch 300 loss 0.082384 acc 0.875000\n",
      "Epoch 101 batch 400 loss 0.102862 acc 0.812500\n",
      "Epoch 101 batch 500 loss 0.076222 acc 0.906250\n",
      "Epoch 101 batch 600 loss 0.096175 acc 0.843750\n",
      "Epoch 101 batch 700 loss 0.088137 acc 0.843750\n",
      "Epoch 101 batch 800 loss 0.078930 acc 0.875000\n",
      "Epoch 101 batch 900 loss 0.071923 acc 0.875000\n",
      "Epoch 101 batch 1000 loss 0.080461 acc 0.843750\n",
      "Epoch 101 batch 1100 loss 0.103364 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 102 batch 0 loss 0.042873 acc 0.937500\n",
      "Epoch 102 batch 100 loss 0.084881 acc 0.843750\n",
      "Epoch 102 batch 200 loss 0.094410 acc 0.812500\n",
      "Epoch 102 batch 300 loss 0.082027 acc 0.875000\n",
      "Epoch 102 batch 400 loss 0.102945 acc 0.812500\n",
      "Epoch 102 batch 500 loss 0.075036 acc 0.906250\n",
      "Epoch 102 batch 600 loss 0.096376 acc 0.843750\n",
      "Epoch 102 batch 700 loss 0.088114 acc 0.843750\n",
      "Epoch 102 batch 800 loss 0.078859 acc 0.875000\n",
      "Epoch 102 batch 900 loss 0.071941 acc 0.875000\n",
      "Epoch 102 batch 1000 loss 0.080474 acc 0.843750\n",
      "Epoch 102 batch 1100 loss 0.103661 acc 0.781250\n",
      "Validation accuracy: 0.831952\n",
      "Epoch 103 batch 0 loss 0.043152 acc 0.937500\n",
      "Epoch 103 batch 100 loss 0.084861 acc 0.843750\n",
      "Epoch 103 batch 200 loss 0.094421 acc 0.812500\n",
      "Epoch 103 batch 300 loss 0.085578 acc 0.875000\n",
      "Epoch 103 batch 400 loss 0.102695 acc 0.812500\n",
      "Epoch 103 batch 500 loss 0.075774 acc 0.906250\n",
      "Epoch 103 batch 600 loss 0.096303 acc 0.843750\n",
      "Epoch 103 batch 700 loss 0.088045 acc 0.843750\n",
      "Epoch 103 batch 800 loss 0.078839 acc 0.875000\n",
      "Epoch 103 batch 900 loss 0.071925 acc 0.875000\n",
      "Epoch 103 batch 1000 loss 0.080412 acc 0.843750\n",
      "Epoch 103 batch 1100 loss 0.102541 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 104 batch 0 loss 0.042856 acc 0.937500\n",
      "Epoch 104 batch 100 loss 0.084925 acc 0.843750\n",
      "Epoch 104 batch 200 loss 0.094475 acc 0.812500\n",
      "Epoch 104 batch 300 loss 0.081513 acc 0.875000\n",
      "Epoch 104 batch 400 loss 0.102942 acc 0.812500\n",
      "Epoch 104 batch 500 loss 0.075773 acc 0.906250\n",
      "Epoch 104 batch 600 loss 0.096296 acc 0.843750\n",
      "Epoch 104 batch 700 loss 0.088148 acc 0.843750\n",
      "Epoch 104 batch 800 loss 0.078833 acc 0.875000\n",
      "Epoch 104 batch 900 loss 0.071978 acc 0.875000\n",
      "Epoch 104 batch 1000 loss 0.080386 acc 0.843750\n",
      "Epoch 104 batch 1100 loss 0.104372 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 105 batch 0 loss 0.040646 acc 0.937500\n",
      "Epoch 105 batch 100 loss 0.085018 acc 0.843750\n",
      "Epoch 105 batch 200 loss 0.094389 acc 0.812500\n",
      "Epoch 105 batch 300 loss 0.083960 acc 0.875000\n",
      "Epoch 105 batch 400 loss 0.102820 acc 0.812500\n",
      "Epoch 105 batch 500 loss 0.075566 acc 0.906250\n",
      "Epoch 105 batch 600 loss 0.096336 acc 0.843750\n",
      "Epoch 105 batch 700 loss 0.088057 acc 0.843750\n",
      "Epoch 105 batch 800 loss 0.078833 acc 0.875000\n",
      "Epoch 105 batch 900 loss 0.071923 acc 0.875000\n",
      "Epoch 105 batch 1000 loss 0.080499 acc 0.843750\n",
      "Epoch 105 batch 1100 loss 0.103409 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 106 batch 0 loss 0.041140 acc 0.937500\n",
      "Epoch 106 batch 100 loss 0.084819 acc 0.843750\n",
      "Epoch 106 batch 200 loss 0.094414 acc 0.812500\n",
      "Epoch 106 batch 300 loss 0.085339 acc 0.875000\n",
      "Epoch 106 batch 400 loss 0.102763 acc 0.812500\n",
      "Epoch 106 batch 500 loss 0.075162 acc 0.906250\n",
      "Epoch 106 batch 600 loss 0.096384 acc 0.843750\n",
      "Epoch 106 batch 700 loss 0.088106 acc 0.843750\n",
      "Epoch 106 batch 800 loss 0.078900 acc 0.875000\n",
      "Epoch 106 batch 900 loss 0.071986 acc 0.875000\n",
      "Epoch 106 batch 1000 loss 0.080458 acc 0.843750\n",
      "Epoch 106 batch 1100 loss 0.105039 acc 0.781250\n",
      "Validation accuracy: 0.830294\n",
      "Epoch 107 batch 0 loss 0.041662 acc 0.937500\n",
      "Epoch 107 batch 100 loss 0.084964 acc 0.843750\n",
      "Epoch 107 batch 200 loss 0.094454 acc 0.812500\n",
      "Epoch 107 batch 300 loss 0.083273 acc 0.875000\n",
      "Epoch 107 batch 400 loss 0.102796 acc 0.812500\n",
      "Epoch 107 batch 500 loss 0.075354 acc 0.906250\n",
      "Epoch 107 batch 600 loss 0.096391 acc 0.843750\n",
      "Epoch 107 batch 700 loss 0.088041 acc 0.843750\n",
      "Epoch 107 batch 800 loss 0.078768 acc 0.875000\n",
      "Epoch 107 batch 900 loss 0.071930 acc 0.875000\n",
      "Epoch 107 batch 1000 loss 0.080509 acc 0.843750\n",
      "Epoch 107 batch 1100 loss 0.103761 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 108 batch 0 loss 0.041262 acc 0.937500\n",
      "Epoch 108 batch 100 loss 0.084726 acc 0.843750\n",
      "Epoch 108 batch 200 loss 0.094408 acc 0.812500\n",
      "Epoch 108 batch 300 loss 0.084897 acc 0.875000\n",
      "Epoch 108 batch 400 loss 0.102773 acc 0.812500\n",
      "Epoch 108 batch 500 loss 0.075329 acc 0.906250\n",
      "Epoch 108 batch 600 loss 0.096364 acc 0.843750\n",
      "Epoch 108 batch 700 loss 0.088095 acc 0.843750\n",
      "Epoch 108 batch 800 loss 0.078760 acc 0.875000\n",
      "Epoch 108 batch 900 loss 0.071932 acc 0.875000\n",
      "Epoch 108 batch 1000 loss 0.080481 acc 0.843750\n",
      "Epoch 108 batch 1100 loss 0.103717 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 109 batch 0 loss 0.040883 acc 0.937500\n",
      "Epoch 109 batch 100 loss 0.084860 acc 0.843750\n",
      "Epoch 109 batch 200 loss 0.094425 acc 0.812500\n",
      "Epoch 109 batch 300 loss 0.081860 acc 0.875000\n",
      "Epoch 109 batch 400 loss 0.102747 acc 0.812500\n",
      "Epoch 109 batch 500 loss 0.075244 acc 0.906250\n",
      "Epoch 109 batch 600 loss 0.096300 acc 0.843750\n",
      "Epoch 109 batch 700 loss 0.088074 acc 0.843750\n",
      "Epoch 109 batch 800 loss 0.078748 acc 0.875000\n",
      "Epoch 109 batch 900 loss 0.071983 acc 0.875000\n",
      "Epoch 109 batch 1000 loss 0.080367 acc 0.843750\n",
      "Epoch 109 batch 1100 loss 0.104477 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 110 batch 0 loss 0.040139 acc 0.937500\n",
      "Epoch 110 batch 100 loss 0.085196 acc 0.843750\n",
      "Epoch 110 batch 200 loss 0.094426 acc 0.812500\n",
      "Epoch 110 batch 300 loss 0.084100 acc 0.875000\n",
      "Epoch 110 batch 400 loss 0.103113 acc 0.812500\n",
      "Epoch 110 batch 500 loss 0.075094 acc 0.906250\n",
      "Epoch 110 batch 600 loss 0.096282 acc 0.843750\n",
      "Epoch 110 batch 700 loss 0.088139 acc 0.843750\n",
      "Epoch 110 batch 800 loss 0.078691 acc 0.875000\n",
      "Epoch 110 batch 900 loss 0.071945 acc 0.875000\n",
      "Epoch 110 batch 1000 loss 0.080463 acc 0.843750\n",
      "Epoch 110 batch 1100 loss 0.107317 acc 0.781250\n",
      "Validation accuracy: 0.832189\n",
      "Epoch 111 batch 0 loss 0.045150 acc 0.937500\n",
      "Epoch 111 batch 100 loss 0.085097 acc 0.843750\n",
      "Epoch 111 batch 200 loss 0.094554 acc 0.812500\n",
      "Epoch 111 batch 300 loss 0.082234 acc 0.875000\n",
      "Epoch 111 batch 400 loss 0.102865 acc 0.812500\n",
      "Epoch 111 batch 500 loss 0.075966 acc 0.906250\n",
      "Epoch 111 batch 600 loss 0.096138 acc 0.843750\n",
      "Epoch 111 batch 700 loss 0.088299 acc 0.843750\n",
      "Epoch 111 batch 800 loss 0.078655 acc 0.875000\n",
      "Epoch 111 batch 900 loss 0.072025 acc 0.875000\n",
      "Epoch 111 batch 1000 loss 0.080434 acc 0.843750\n",
      "Epoch 111 batch 1100 loss 0.103863 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 112 batch 0 loss 0.043615 acc 0.937500\n",
      "Epoch 112 batch 100 loss 0.084982 acc 0.843750\n",
      "Epoch 112 batch 200 loss 0.094491 acc 0.812500\n",
      "Epoch 112 batch 300 loss 0.083863 acc 0.875000\n",
      "Epoch 112 batch 400 loss 0.103049 acc 0.812500\n",
      "Epoch 112 batch 500 loss 0.075156 acc 0.906250\n",
      "Epoch 112 batch 600 loss 0.096466 acc 0.843750\n",
      "Epoch 112 batch 700 loss 0.088072 acc 0.843750\n",
      "Epoch 112 batch 800 loss 0.078657 acc 0.875000\n",
      "Epoch 112 batch 900 loss 0.071932 acc 0.875000\n",
      "Epoch 112 batch 1000 loss 0.080463 acc 0.843750\n",
      "Epoch 112 batch 1100 loss 0.103570 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 113 batch 0 loss 0.042998 acc 0.937500\n",
      "Epoch 113 batch 100 loss 0.085041 acc 0.843750\n",
      "Epoch 113 batch 200 loss 0.094505 acc 0.812500\n",
      "Epoch 113 batch 300 loss 0.084176 acc 0.875000\n",
      "Epoch 113 batch 400 loss 0.103019 acc 0.812500\n",
      "Epoch 113 batch 500 loss 0.075093 acc 0.906250\n",
      "Epoch 113 batch 600 loss 0.096471 acc 0.843750\n",
      "Epoch 113 batch 700 loss 0.088029 acc 0.843750\n",
      "Epoch 113 batch 800 loss 0.078633 acc 0.875000\n",
      "Epoch 113 batch 900 loss 0.071935 acc 0.875000\n",
      "Epoch 113 batch 1000 loss 0.080383 acc 0.843750\n",
      "Epoch 113 batch 1100 loss 0.106375 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.831873\n",
      "Epoch 114 batch 0 loss 0.043205 acc 0.937500\n",
      "Epoch 114 batch 100 loss 0.085222 acc 0.843750\n",
      "Epoch 114 batch 200 loss 0.094540 acc 0.812500\n",
      "Epoch 114 batch 300 loss 0.083840 acc 0.875000\n",
      "Epoch 114 batch 400 loss 0.102917 acc 0.812500\n",
      "Epoch 114 batch 500 loss 0.075200 acc 0.906250\n",
      "Epoch 114 batch 600 loss 0.096403 acc 0.843750\n",
      "Epoch 114 batch 700 loss 0.088054 acc 0.843750\n",
      "Epoch 114 batch 800 loss 0.078611 acc 0.875000\n",
      "Epoch 114 batch 900 loss 0.071934 acc 0.875000\n",
      "Epoch 114 batch 1000 loss 0.080510 acc 0.843750\n",
      "Epoch 114 batch 1100 loss 0.104297 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 115 batch 0 loss 0.040899 acc 0.937500\n",
      "Epoch 115 batch 100 loss 0.085062 acc 0.843750\n",
      "Epoch 115 batch 200 loss 0.094469 acc 0.812500\n",
      "Epoch 115 batch 300 loss 0.083599 acc 0.875000\n",
      "Epoch 115 batch 400 loss 0.102881 acc 0.812500\n",
      "Epoch 115 batch 500 loss 0.075354 acc 0.906250\n",
      "Epoch 115 batch 600 loss 0.096262 acc 0.843750\n",
      "Epoch 115 batch 700 loss 0.088059 acc 0.843750\n",
      "Epoch 115 batch 800 loss 0.078632 acc 0.875000\n",
      "Epoch 115 batch 900 loss 0.071933 acc 0.875000\n",
      "Epoch 115 batch 1000 loss 0.080517 acc 0.843750\n",
      "Epoch 115 batch 1100 loss 0.103684 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 116 batch 0 loss 0.040852 acc 0.937500\n",
      "Epoch 116 batch 100 loss 0.084945 acc 0.843750\n",
      "Epoch 116 batch 200 loss 0.094464 acc 0.812500\n",
      "Epoch 116 batch 300 loss 0.083664 acc 0.875000\n",
      "Epoch 116 batch 400 loss 0.103034 acc 0.812500\n",
      "Epoch 116 batch 500 loss 0.075103 acc 0.906250\n",
      "Epoch 116 batch 600 loss 0.096417 acc 0.843750\n",
      "Epoch 116 batch 700 loss 0.088027 acc 0.843750\n",
      "Epoch 116 batch 800 loss 0.078627 acc 0.875000\n",
      "Epoch 116 batch 900 loss 0.071938 acc 0.875000\n",
      "Epoch 116 batch 1000 loss 0.080474 acc 0.843750\n",
      "Epoch 116 batch 1100 loss 0.103459 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 117 batch 0 loss 0.044372 acc 0.937500\n",
      "Epoch 117 batch 100 loss 0.085195 acc 0.843750\n",
      "Epoch 117 batch 200 loss 0.094566 acc 0.812500\n",
      "Epoch 117 batch 300 loss 0.084484 acc 0.875000\n",
      "Epoch 117 batch 400 loss 0.103014 acc 0.812500\n",
      "Epoch 117 batch 500 loss 0.075023 acc 0.906250\n",
      "Epoch 117 batch 600 loss 0.096442 acc 0.843750\n",
      "Epoch 117 batch 700 loss 0.088038 acc 0.843750\n",
      "Epoch 117 batch 800 loss 0.078549 acc 0.875000\n",
      "Epoch 117 batch 900 loss 0.072019 acc 0.875000\n",
      "Epoch 117 batch 1000 loss 0.080483 acc 0.843750\n",
      "Epoch 117 batch 1100 loss 0.104038 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 118 batch 0 loss 0.041095 acc 0.937500\n",
      "Epoch 118 batch 100 loss 0.084949 acc 0.843750\n",
      "Epoch 118 batch 200 loss 0.094482 acc 0.812500\n",
      "Epoch 118 batch 300 loss 0.083643 acc 0.875000\n",
      "Epoch 118 batch 400 loss 0.102887 acc 0.812500\n",
      "Epoch 118 batch 500 loss 0.075226 acc 0.906250\n",
      "Epoch 118 batch 600 loss 0.096301 acc 0.843750\n",
      "Epoch 118 batch 700 loss 0.088085 acc 0.843750\n",
      "Epoch 118 batch 800 loss 0.078616 acc 0.875000\n",
      "Epoch 118 batch 900 loss 0.071933 acc 0.875000\n",
      "Epoch 118 batch 1000 loss 0.080500 acc 0.843750\n",
      "Epoch 118 batch 1100 loss 0.104056 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 119 batch 0 loss 0.043413 acc 0.937500\n",
      "Epoch 119 batch 100 loss 0.085164 acc 0.843750\n",
      "Epoch 119 batch 200 loss 0.094542 acc 0.812500\n",
      "Epoch 119 batch 300 loss 0.082887 acc 0.875000\n",
      "Epoch 119 batch 400 loss 0.102923 acc 0.812500\n",
      "Epoch 119 batch 500 loss 0.075089 acc 0.906250\n",
      "Epoch 119 batch 600 loss 0.096376 acc 0.843750\n",
      "Epoch 119 batch 700 loss 0.088012 acc 0.843750\n",
      "Epoch 119 batch 800 loss 0.078584 acc 0.875000\n",
      "Epoch 119 batch 900 loss 0.071934 acc 0.875000\n",
      "Epoch 119 batch 1000 loss 0.080537 acc 0.843750\n",
      "Epoch 119 batch 1100 loss 0.103643 acc 0.781250\n",
      "Validation accuracy: 0.831321\n",
      "Epoch 120 batch 0 loss 0.042275 acc 0.937500\n",
      "Epoch 120 batch 100 loss 0.085102 acc 0.843750\n",
      "Epoch 120 batch 200 loss 0.094554 acc 0.812500\n",
      "Epoch 120 batch 300 loss 0.084281 acc 0.875000\n",
      "Epoch 120 batch 400 loss 0.103127 acc 0.812500\n",
      "Epoch 120 batch 500 loss 0.075112 acc 0.906250\n",
      "Epoch 120 batch 600 loss 0.096387 acc 0.843750\n",
      "Epoch 120 batch 700 loss 0.088048 acc 0.843750\n",
      "Epoch 120 batch 800 loss 0.078558 acc 0.875000\n",
      "Epoch 120 batch 900 loss 0.071940 acc 0.875000\n",
      "Epoch 120 batch 1000 loss 0.080475 acc 0.843750\n",
      "Epoch 120 batch 1100 loss 0.103403 acc 0.781250\n",
      "Validation accuracy: 0.830294\n",
      "Epoch 121 batch 0 loss 0.041448 acc 0.937500\n",
      "Epoch 121 batch 100 loss 0.085075 acc 0.843750\n",
      "Epoch 121 batch 200 loss 0.094535 acc 0.812500\n",
      "Epoch 121 batch 300 loss 0.081639 acc 0.875000\n",
      "Epoch 121 batch 400 loss 0.103017 acc 0.812500\n",
      "Epoch 121 batch 500 loss 0.074525 acc 0.906250\n",
      "Epoch 121 batch 600 loss 0.096343 acc 0.843750\n",
      "Epoch 121 batch 700 loss 0.088178 acc 0.843750\n",
      "Epoch 121 batch 800 loss 0.078526 acc 0.875000\n",
      "Epoch 121 batch 900 loss 0.071951 acc 0.875000\n",
      "Epoch 121 batch 1000 loss 0.080497 acc 0.843750\n",
      "Epoch 121 batch 1100 loss 0.104590 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 122 batch 0 loss 0.040491 acc 0.937500\n",
      "Epoch 122 batch 100 loss 0.085479 acc 0.843750\n",
      "Epoch 122 batch 200 loss 0.094532 acc 0.812500\n",
      "Epoch 122 batch 300 loss 0.081650 acc 0.875000\n",
      "Epoch 122 batch 400 loss 0.103011 acc 0.812500\n",
      "Epoch 122 batch 500 loss 0.075038 acc 0.906250\n",
      "Epoch 122 batch 600 loss 0.096371 acc 0.843750\n",
      "Epoch 122 batch 700 loss 0.088049 acc 0.843750\n",
      "Epoch 122 batch 800 loss 0.078551 acc 0.875000\n",
      "Epoch 122 batch 900 loss 0.071946 acc 0.875000\n",
      "Epoch 122 batch 1000 loss 0.080489 acc 0.843750\n",
      "Epoch 122 batch 1100 loss 0.103321 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 123 batch 0 loss 0.041120 acc 0.937500\n",
      "Epoch 123 batch 100 loss 0.085063 acc 0.843750\n",
      "Epoch 123 batch 200 loss 0.094562 acc 0.812500\n",
      "Epoch 123 batch 300 loss 0.081440 acc 0.875000\n",
      "Epoch 123 batch 400 loss 0.103033 acc 0.812500\n",
      "Epoch 123 batch 500 loss 0.075269 acc 0.906250\n",
      "Epoch 123 batch 600 loss 0.096245 acc 0.843750\n",
      "Epoch 123 batch 700 loss 0.088116 acc 0.843750\n",
      "Epoch 123 batch 800 loss 0.078530 acc 0.875000\n",
      "Epoch 123 batch 900 loss 0.071959 acc 0.875000\n",
      "Epoch 123 batch 1000 loss 0.080432 acc 0.843750\n",
      "Epoch 123 batch 1100 loss 0.104336 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 124 batch 0 loss 0.042679 acc 0.937500\n",
      "Epoch 124 batch 100 loss 0.085182 acc 0.843750\n",
      "Epoch 124 batch 200 loss 0.094547 acc 0.812500\n",
      "Epoch 124 batch 300 loss 0.081191 acc 0.875000\n",
      "Epoch 124 batch 400 loss 0.103109 acc 0.812500\n",
      "Epoch 124 batch 500 loss 0.075295 acc 0.906250\n",
      "Epoch 124 batch 600 loss 0.096236 acc 0.843750\n",
      "Epoch 124 batch 700 loss 0.088181 acc 0.843750\n",
      "Epoch 124 batch 800 loss 0.078513 acc 0.875000\n",
      "Epoch 124 batch 900 loss 0.071996 acc 0.875000\n",
      "Epoch 124 batch 1000 loss 0.080395 acc 0.843750\n",
      "Epoch 124 batch 1100 loss 0.102911 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 125 batch 0 loss 0.044663 acc 0.937500\n",
      "Epoch 125 batch 100 loss 0.085040 acc 0.843750\n",
      "Epoch 125 batch 200 loss 0.094617 acc 0.812500\n",
      "Epoch 125 batch 300 loss 0.082801 acc 0.875000\n",
      "Epoch 125 batch 400 loss 0.103204 acc 0.812500\n",
      "Epoch 125 batch 500 loss 0.074595 acc 0.906250\n",
      "Epoch 125 batch 600 loss 0.096512 acc 0.843750\n",
      "Epoch 125 batch 700 loss 0.088110 acc 0.843750\n",
      "Epoch 125 batch 800 loss 0.078612 acc 0.875000\n",
      "Epoch 125 batch 900 loss 0.071954 acc 0.875000\n",
      "Epoch 125 batch 1000 loss 0.080493 acc 0.843750\n",
      "Epoch 125 batch 1100 loss 0.103732 acc 0.781250\n",
      "Validation accuracy: 0.830689\n",
      "Epoch 126 batch 0 loss 0.041746 acc 0.937500\n",
      "Epoch 126 batch 100 loss 0.085187 acc 0.843750\n",
      "Epoch 126 batch 200 loss 0.094570 acc 0.812500\n",
      "Epoch 126 batch 300 loss 0.082871 acc 0.875000\n",
      "Epoch 126 batch 400 loss 0.103216 acc 0.812500\n",
      "Epoch 126 batch 500 loss 0.074547 acc 0.906250\n",
      "Epoch 126 batch 600 loss 0.096606 acc 0.843750\n",
      "Epoch 126 batch 700 loss 0.088011 acc 0.843750\n",
      "Epoch 126 batch 800 loss 0.078461 acc 0.875000\n",
      "Epoch 126 batch 900 loss 0.071941 acc 0.875000\n",
      "Epoch 126 batch 1000 loss 0.080390 acc 0.843750\n",
      "Epoch 126 batch 1100 loss 0.104068 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 127 batch 0 loss 0.041093 acc 0.937500\n",
      "Epoch 127 batch 100 loss 0.085135 acc 0.843750\n",
      "Epoch 127 batch 200 loss 0.094523 acc 0.812500\n",
      "Epoch 127 batch 300 loss 0.084720 acc 0.875000\n",
      "Epoch 127 batch 400 loss 0.102967 acc 0.812500\n",
      "Epoch 127 batch 500 loss 0.074816 acc 0.906250\n",
      "Epoch 127 batch 600 loss 0.096409 acc 0.843750\n",
      "Epoch 127 batch 700 loss 0.088042 acc 0.843750\n",
      "Epoch 127 batch 800 loss 0.078475 acc 0.875000\n",
      "Epoch 127 batch 900 loss 0.071942 acc 0.875000\n",
      "Epoch 127 batch 1000 loss 0.080546 acc 0.843750\n",
      "Epoch 127 batch 1100 loss 0.103718 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.829900\n",
      "Epoch 128 batch 0 loss 0.041326 acc 0.937500\n",
      "Epoch 128 batch 100 loss 0.085114 acc 0.843750\n",
      "Epoch 128 batch 200 loss 0.094556 acc 0.812500\n",
      "Epoch 128 batch 300 loss 0.082522 acc 0.875000\n",
      "Epoch 128 batch 400 loss 0.102998 acc 0.812500\n",
      "Epoch 128 batch 500 loss 0.074875 acc 0.906250\n",
      "Epoch 128 batch 600 loss 0.096381 acc 0.843750\n",
      "Epoch 128 batch 700 loss 0.088054 acc 0.843750\n",
      "Epoch 128 batch 800 loss 0.078482 acc 0.875000\n",
      "Epoch 128 batch 900 loss 0.071944 acc 0.875000\n",
      "Epoch 128 batch 1000 loss 0.080499 acc 0.843750\n",
      "Epoch 128 batch 1100 loss 0.104875 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 129 batch 0 loss 0.040500 acc 0.937500\n",
      "Epoch 129 batch 100 loss 0.085374 acc 0.843750\n",
      "Epoch 129 batch 200 loss 0.094541 acc 0.812500\n",
      "Epoch 129 batch 300 loss 0.086118 acc 0.875000\n",
      "Epoch 129 batch 400 loss 0.102867 acc 0.812500\n",
      "Epoch 129 batch 500 loss 0.074625 acc 0.906250\n",
      "Epoch 129 batch 600 loss 0.096452 acc 0.843750\n",
      "Epoch 129 batch 700 loss 0.087980 acc 0.843750\n",
      "Epoch 129 batch 800 loss 0.078423 acc 0.875000\n",
      "Epoch 129 batch 900 loss 0.071948 acc 0.875000\n",
      "Epoch 129 batch 1000 loss 0.080581 acc 0.843750\n",
      "Epoch 129 batch 1100 loss 0.103091 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 130 batch 0 loss 0.041425 acc 0.937500\n",
      "Epoch 130 batch 100 loss 0.085135 acc 0.843750\n",
      "Epoch 130 batch 200 loss 0.094572 acc 0.812500\n",
      "Epoch 130 batch 300 loss 0.081747 acc 0.875000\n",
      "Epoch 130 batch 400 loss 0.102954 acc 0.812500\n",
      "Epoch 130 batch 500 loss 0.074689 acc 0.906250\n",
      "Epoch 130 batch 600 loss 0.096428 acc 0.843750\n",
      "Epoch 130 batch 700 loss 0.088054 acc 0.843750\n",
      "Epoch 130 batch 800 loss 0.078485 acc 0.875000\n",
      "Epoch 130 batch 900 loss 0.071969 acc 0.875000\n",
      "Epoch 130 batch 1000 loss 0.080514 acc 0.843750\n",
      "Epoch 130 batch 1100 loss 0.103788 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 131 batch 0 loss 0.041036 acc 0.937500\n",
      "Epoch 131 batch 100 loss 0.085196 acc 0.843750\n",
      "Epoch 131 batch 200 loss 0.094583 acc 0.812500\n",
      "Epoch 131 batch 300 loss 0.081815 acc 0.875000\n",
      "Epoch 131 batch 400 loss 0.102967 acc 0.812500\n",
      "Epoch 131 batch 500 loss 0.075140 acc 0.906250\n",
      "Epoch 131 batch 600 loss 0.096262 acc 0.843750\n",
      "Epoch 131 batch 700 loss 0.088075 acc 0.843750\n",
      "Epoch 131 batch 800 loss 0.078439 acc 0.875000\n",
      "Epoch 131 batch 900 loss 0.071975 acc 0.875000\n",
      "Epoch 131 batch 1000 loss 0.080343 acc 0.843750\n",
      "Epoch 131 batch 1100 loss 0.104226 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 132 batch 0 loss 0.040178 acc 0.937500\n",
      "Epoch 132 batch 100 loss 0.085504 acc 0.843750\n",
      "Epoch 132 batch 200 loss 0.094544 acc 0.812500\n",
      "Epoch 132 batch 300 loss 0.081620 acc 0.875000\n",
      "Epoch 132 batch 400 loss 0.103050 acc 0.812500\n",
      "Epoch 132 batch 500 loss 0.074744 acc 0.906250\n",
      "Epoch 132 batch 600 loss 0.096382 acc 0.843750\n",
      "Epoch 132 batch 700 loss 0.087974 acc 0.843750\n",
      "Epoch 132 batch 800 loss 0.078456 acc 0.875000\n",
      "Epoch 132 batch 900 loss 0.071942 acc 0.875000\n",
      "Epoch 132 batch 1000 loss 0.080455 acc 0.843750\n",
      "Epoch 132 batch 1100 loss 0.104249 acc 0.781250\n",
      "Validation accuracy: 0.830768\n",
      "Epoch 133 batch 0 loss 0.041505 acc 0.937500\n",
      "Epoch 133 batch 100 loss 0.085119 acc 0.843750\n",
      "Epoch 133 batch 200 loss 0.094553 acc 0.812500\n",
      "Epoch 133 batch 300 loss 0.083159 acc 0.875000\n",
      "Epoch 133 batch 400 loss 0.103056 acc 0.812500\n",
      "Epoch 133 batch 500 loss 0.075018 acc 0.906250\n",
      "Epoch 133 batch 600 loss 0.096305 acc 0.843750\n",
      "Epoch 133 batch 700 loss 0.088092 acc 0.843750\n",
      "Epoch 133 batch 800 loss 0.078441 acc 0.875000\n",
      "Epoch 133 batch 900 loss 0.071941 acc 0.875000\n",
      "Epoch 133 batch 1000 loss 0.080524 acc 0.843750\n",
      "Epoch 133 batch 1100 loss 0.104093 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 134 batch 0 loss 0.041161 acc 0.937500\n",
      "Epoch 134 batch 100 loss 0.085128 acc 0.843750\n",
      "Epoch 134 batch 200 loss 0.094555 acc 0.812500\n",
      "Epoch 134 batch 300 loss 0.085119 acc 0.875000\n",
      "Epoch 134 batch 400 loss 0.102979 acc 0.812500\n",
      "Epoch 134 batch 500 loss 0.074435 acc 0.906250\n",
      "Epoch 134 batch 600 loss 0.096584 acc 0.843750\n",
      "Epoch 134 batch 700 loss 0.087945 acc 0.843750\n",
      "Epoch 134 batch 800 loss 0.078386 acc 0.875000\n",
      "Epoch 134 batch 900 loss 0.071953 acc 0.875000\n",
      "Epoch 134 batch 1000 loss 0.080499 acc 0.843750\n",
      "Epoch 134 batch 1100 loss 0.102679 acc 0.781250\n",
      "Validation accuracy: 0.831163\n",
      "Epoch 135 batch 0 loss 0.042425 acc 0.937500\n",
      "Epoch 135 batch 100 loss 0.085425 acc 0.843750\n",
      "Epoch 135 batch 200 loss 0.094663 acc 0.812500\n",
      "Epoch 135 batch 300 loss 0.083219 acc 0.875000\n",
      "Epoch 135 batch 400 loss 0.102825 acc 0.812500\n",
      "Epoch 135 batch 500 loss 0.075016 acc 0.906250\n",
      "Epoch 135 batch 600 loss 0.096288 acc 0.843750\n",
      "Epoch 135 batch 700 loss 0.088146 acc 0.843750\n",
      "Epoch 135 batch 800 loss 0.078364 acc 0.875000\n",
      "Epoch 135 batch 900 loss 0.072032 acc 0.875000\n",
      "Epoch 135 batch 1000 loss 0.080404 acc 0.843750\n",
      "Epoch 135 batch 1100 loss 0.103844 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 136 batch 0 loss 0.041434 acc 0.937500\n",
      "Epoch 136 batch 100 loss 0.085094 acc 0.843750\n",
      "Epoch 136 batch 200 loss 0.094617 acc 0.812500\n",
      "Epoch 136 batch 300 loss 0.084307 acc 0.875000\n",
      "Epoch 136 batch 400 loss 0.102922 acc 0.812500\n",
      "Epoch 136 batch 500 loss 0.074974 acc 0.906250\n",
      "Epoch 136 batch 600 loss 0.096384 acc 0.843750\n",
      "Epoch 136 batch 700 loss 0.088180 acc 0.843750\n",
      "Epoch 136 batch 800 loss 0.078324 acc 0.875000\n",
      "Epoch 136 batch 900 loss 0.071955 acc 0.875000\n",
      "Epoch 136 batch 1000 loss 0.080347 acc 0.843750\n",
      "Epoch 136 batch 1100 loss 0.103081 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 137 batch 0 loss 0.043605 acc 0.937500\n",
      "Epoch 137 batch 100 loss 0.085327 acc 0.843750\n",
      "Epoch 137 batch 200 loss 0.094687 acc 0.812500\n",
      "Epoch 137 batch 300 loss 0.081477 acc 0.875000\n",
      "Epoch 137 batch 400 loss 0.102892 acc 0.812500\n",
      "Epoch 137 batch 500 loss 0.075087 acc 0.906250\n",
      "Epoch 137 batch 600 loss 0.096261 acc 0.843750\n",
      "Epoch 137 batch 700 loss 0.088299 acc 0.843750\n",
      "Epoch 137 batch 800 loss 0.078657 acc 0.875000\n",
      "Epoch 137 batch 900 loss 0.071993 acc 0.875000\n",
      "Epoch 137 batch 1000 loss 0.080412 acc 0.843750\n",
      "Epoch 137 batch 1100 loss 0.104634 acc 0.781250\n",
      "Validation accuracy: 0.830689\n",
      "Epoch 138 batch 0 loss 0.041888 acc 0.937500\n",
      "Epoch 138 batch 100 loss 0.085068 acc 0.843750\n",
      "Epoch 138 batch 200 loss 0.094618 acc 0.812500\n",
      "Epoch 138 batch 300 loss 0.081846 acc 0.875000\n",
      "Epoch 138 batch 400 loss 0.102837 acc 0.812500\n",
      "Epoch 138 batch 500 loss 0.074907 acc 0.906250\n",
      "Epoch 138 batch 600 loss 0.096343 acc 0.843750\n",
      "Epoch 138 batch 700 loss 0.088131 acc 0.843750\n",
      "Epoch 138 batch 800 loss 0.078473 acc 0.875000\n",
      "Epoch 138 batch 900 loss 0.072025 acc 0.875000\n",
      "Epoch 138 batch 1000 loss 0.080431 acc 0.843750\n",
      "Epoch 138 batch 1100 loss 0.104584 acc 0.781250\n",
      "Validation accuracy: 0.831478\n",
      "Epoch 139 batch 0 loss 0.042370 acc 0.937500\n",
      "Epoch 139 batch 100 loss 0.085222 acc 0.843750\n",
      "Epoch 139 batch 200 loss 0.094633 acc 0.812500\n",
      "Epoch 139 batch 300 loss 0.083793 acc 0.875000\n",
      "Epoch 139 batch 400 loss 0.102864 acc 0.812500\n",
      "Epoch 139 batch 500 loss 0.074841 acc 0.906250\n",
      "Epoch 139 batch 600 loss 0.096358 acc 0.843750\n",
      "Epoch 139 batch 700 loss 0.088097 acc 0.843750\n",
      "Epoch 139 batch 800 loss 0.078325 acc 0.875000\n",
      "Epoch 139 batch 900 loss 0.071933 acc 0.875000\n",
      "Epoch 139 batch 1000 loss 0.080504 acc 0.843750\n",
      "Epoch 139 batch 1100 loss 0.103074 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 140 batch 0 loss 0.041599 acc 0.937500\n",
      "Epoch 140 batch 100 loss 0.085188 acc 0.843750\n",
      "Epoch 140 batch 200 loss 0.094650 acc 0.812500\n",
      "Epoch 140 batch 300 loss 0.082395 acc 0.875000\n",
      "Epoch 140 batch 400 loss 0.102909 acc 0.812500\n",
      "Epoch 140 batch 500 loss 0.074941 acc 0.906250\n",
      "Epoch 140 batch 600 loss 0.096303 acc 0.843750\n",
      "Epoch 140 batch 700 loss 0.088164 acc 0.843750\n",
      "Epoch 140 batch 800 loss 0.078354 acc 0.875000\n",
      "Epoch 140 batch 900 loss 0.071936 acc 0.875000\n",
      "Epoch 140 batch 1000 loss 0.080455 acc 0.843750\n",
      "Epoch 140 batch 1100 loss 0.104119 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 141 batch 0 loss 0.043738 acc 0.937500\n",
      "Epoch 141 batch 100 loss 0.085206 acc 0.843750\n",
      "Epoch 141 batch 200 loss 0.094600 acc 0.812500\n",
      "Epoch 141 batch 300 loss 0.083987 acc 0.875000\n",
      "Epoch 141 batch 400 loss 0.103017 acc 0.812500\n",
      "Epoch 141 batch 500 loss 0.074364 acc 0.906250\n",
      "Epoch 141 batch 600 loss 0.096397 acc 0.843750\n",
      "Epoch 141 batch 700 loss 0.088171 acc 0.843750\n",
      "Epoch 141 batch 800 loss 0.078276 acc 0.875000\n",
      "Epoch 141 batch 900 loss 0.071953 acc 0.875000\n",
      "Epoch 141 batch 1000 loss 0.080351 acc 0.843750\n",
      "Epoch 141 batch 1100 loss 0.104644 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.829900\n",
      "Epoch 142 batch 0 loss 0.040512 acc 0.937500\n",
      "Epoch 142 batch 100 loss 0.085423 acc 0.843750\n",
      "Epoch 142 batch 200 loss 0.094607 acc 0.812500\n",
      "Epoch 142 batch 300 loss 0.081069 acc 0.875000\n",
      "Epoch 142 batch 400 loss 0.103097 acc 0.812500\n",
      "Epoch 142 batch 500 loss 0.074970 acc 0.906250\n",
      "Epoch 142 batch 600 loss 0.096310 acc 0.843750\n",
      "Epoch 142 batch 700 loss 0.088121 acc 0.843750\n",
      "Epoch 142 batch 800 loss 0.078335 acc 0.875000\n",
      "Epoch 142 batch 900 loss 0.071937 acc 0.875000\n",
      "Epoch 142 batch 1000 loss 0.080399 acc 0.843750\n",
      "Epoch 142 batch 1100 loss 0.103669 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 143 batch 0 loss 0.040671 acc 0.937500\n",
      "Epoch 143 batch 100 loss 0.085246 acc 0.843750\n",
      "Epoch 143 batch 200 loss 0.094590 acc 0.812500\n",
      "Epoch 143 batch 300 loss 0.084297 acc 0.875000\n",
      "Epoch 143 batch 400 loss 0.102956 acc 0.812500\n",
      "Epoch 143 batch 500 loss 0.074719 acc 0.906250\n",
      "Epoch 143 batch 600 loss 0.096365 acc 0.843750\n",
      "Epoch 143 batch 700 loss 0.088049 acc 0.843750\n",
      "Epoch 143 batch 800 loss 0.078322 acc 0.875000\n",
      "Epoch 143 batch 900 loss 0.071935 acc 0.875000\n",
      "Epoch 143 batch 1000 loss 0.080450 acc 0.843750\n",
      "Epoch 143 batch 1100 loss 0.103780 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 144 batch 0 loss 0.041096 acc 0.937500\n",
      "Epoch 144 batch 100 loss 0.085206 acc 0.843750\n",
      "Epoch 144 batch 200 loss 0.094620 acc 0.812500\n",
      "Epoch 144 batch 300 loss 0.083449 acc 0.875000\n",
      "Epoch 144 batch 400 loss 0.103035 acc 0.812500\n",
      "Epoch 144 batch 500 loss 0.074719 acc 0.906250\n",
      "Epoch 144 batch 600 loss 0.096373 acc 0.843750\n",
      "Epoch 144 batch 700 loss 0.088085 acc 0.843750\n",
      "Epoch 144 batch 800 loss 0.078307 acc 0.875000\n",
      "Epoch 144 batch 900 loss 0.071981 acc 0.875000\n",
      "Epoch 144 batch 1000 loss 0.080355 acc 0.843750\n",
      "Epoch 144 batch 1100 loss 0.105154 acc 0.781250\n",
      "Validation accuracy: 0.830689\n",
      "Epoch 145 batch 0 loss 0.041789 acc 0.937500\n",
      "Epoch 145 batch 100 loss 0.085363 acc 0.843750\n",
      "Epoch 145 batch 200 loss 0.094647 acc 0.812500\n",
      "Epoch 145 batch 300 loss 0.086126 acc 0.875000\n",
      "Epoch 145 batch 400 loss 0.102954 acc 0.812500\n",
      "Epoch 145 batch 500 loss 0.074475 acc 0.906250\n",
      "Epoch 145 batch 600 loss 0.096422 acc 0.843750\n",
      "Epoch 145 batch 700 loss 0.088062 acc 0.843750\n",
      "Epoch 145 batch 800 loss 0.078232 acc 0.875000\n",
      "Epoch 145 batch 900 loss 0.071952 acc 0.875000\n",
      "Epoch 145 batch 1000 loss 0.080551 acc 0.843750\n",
      "Epoch 145 batch 1100 loss 0.103281 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 146 batch 0 loss 0.043664 acc 0.937500\n",
      "Epoch 146 batch 100 loss 0.085553 acc 0.843750\n",
      "Epoch 146 batch 200 loss 0.094727 acc 0.812500\n",
      "Epoch 146 batch 300 loss 0.083322 acc 0.875000\n",
      "Epoch 146 batch 400 loss 0.102932 acc 0.812500\n",
      "Epoch 146 batch 500 loss 0.073855 acc 0.906250\n",
      "Epoch 146 batch 600 loss 0.096512 acc 0.843750\n",
      "Epoch 146 batch 700 loss 0.088223 acc 0.843750\n",
      "Epoch 146 batch 800 loss 0.078334 acc 0.875000\n",
      "Epoch 146 batch 900 loss 0.071991 acc 0.875000\n",
      "Epoch 146 batch 1000 loss 0.080450 acc 0.843750\n",
      "Epoch 146 batch 1100 loss 0.104857 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 147 batch 0 loss 0.043492 acc 0.937500\n",
      "Epoch 147 batch 100 loss 0.085490 acc 0.843750\n",
      "Epoch 147 batch 200 loss 0.094708 acc 0.812500\n",
      "Epoch 147 batch 300 loss 0.082414 acc 0.875000\n",
      "Epoch 147 batch 400 loss 0.103016 acc 0.812500\n",
      "Epoch 147 batch 500 loss 0.074683 acc 0.906250\n",
      "Epoch 147 batch 600 loss 0.096408 acc 0.843750\n",
      "Epoch 147 batch 700 loss 0.088058 acc 0.843750\n",
      "Epoch 147 batch 800 loss 0.078260 acc 0.875000\n",
      "Epoch 147 batch 900 loss 0.071940 acc 0.875000\n",
      "Epoch 147 batch 1000 loss 0.080517 acc 0.843750\n",
      "Epoch 147 batch 1100 loss 0.103911 acc 0.781250\n",
      "Validation accuracy: 0.830373\n",
      "Epoch 148 batch 0 loss 0.041311 acc 0.937500\n",
      "Epoch 148 batch 100 loss 0.085244 acc 0.843750\n",
      "Epoch 148 batch 200 loss 0.094637 acc 0.812500\n",
      "Epoch 148 batch 300 loss 0.085297 acc 0.875000\n",
      "Epoch 148 batch 400 loss 0.102950 acc 0.812500\n",
      "Epoch 148 batch 500 loss 0.074686 acc 0.906250\n",
      "Epoch 148 batch 600 loss 0.096313 acc 0.843750\n",
      "Epoch 148 batch 700 loss 0.088162 acc 0.843750\n",
      "Epoch 148 batch 800 loss 0.078214 acc 0.875000\n",
      "Epoch 148 batch 900 loss 0.071957 acc 0.875000\n",
      "Epoch 148 batch 1000 loss 0.080475 acc 0.843750\n",
      "Epoch 148 batch 1100 loss 0.104001 acc 0.781250\n",
      "Validation accuracy: 0.829821\n",
      "Epoch 149 batch 0 loss 0.041015 acc 0.937500\n",
      "Epoch 149 batch 100 loss 0.085241 acc 0.843750\n",
      "Epoch 149 batch 200 loss 0.094625 acc 0.812500\n",
      "Epoch 149 batch 300 loss 0.082312 acc 0.875000\n",
      "Epoch 149 batch 400 loss 0.103126 acc 0.812500\n",
      "Epoch 149 batch 500 loss 0.074695 acc 0.906250\n",
      "Epoch 149 batch 600 loss 0.096368 acc 0.843750\n",
      "Epoch 149 batch 700 loss 0.088090 acc 0.843750\n",
      "Epoch 149 batch 800 loss 0.078271 acc 0.875000\n",
      "Epoch 149 batch 900 loss 0.071940 acc 0.875000\n",
      "Epoch 149 batch 1000 loss 0.080463 acc 0.843750\n",
      "Epoch 149 batch 1100 loss 0.103628 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 150 batch 0 loss 0.043898 acc 0.937500\n",
      "Epoch 150 batch 100 loss 0.085389 acc 0.843750\n",
      "Epoch 150 batch 200 loss 0.094677 acc 0.812500\n",
      "Epoch 150 batch 300 loss 0.082275 acc 0.875000\n",
      "Epoch 150 batch 400 loss 0.102768 acc 0.812500\n",
      "Epoch 150 batch 500 loss 0.074324 acc 0.906250\n",
      "Epoch 150 batch 600 loss 0.096463 acc 0.843750\n",
      "Epoch 150 batch 700 loss 0.088046 acc 0.843750\n",
      "Epoch 150 batch 800 loss 0.078309 acc 0.875000\n",
      "Epoch 150 batch 900 loss 0.071958 acc 0.875000\n",
      "Epoch 150 batch 1000 loss 0.080459 acc 0.843750\n",
      "Epoch 150 batch 1100 loss 0.104421 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 151 batch 0 loss 0.040458 acc 0.937500\n",
      "Epoch 151 batch 100 loss 0.085446 acc 0.843750\n",
      "Epoch 151 batch 200 loss 0.094627 acc 0.812500\n",
      "Epoch 151 batch 300 loss 0.081593 acc 0.875000\n",
      "Epoch 151 batch 400 loss 0.102899 acc 0.812500\n",
      "Epoch 151 batch 500 loss 0.074797 acc 0.906250\n",
      "Epoch 151 batch 600 loss 0.096293 acc 0.843750\n",
      "Epoch 151 batch 700 loss 0.088110 acc 0.843750\n",
      "Epoch 151 batch 800 loss 0.078350 acc 0.875000\n",
      "Epoch 151 batch 900 loss 0.071959 acc 0.875000\n",
      "Epoch 151 batch 1000 loss 0.080458 acc 0.843750\n",
      "Epoch 151 batch 1100 loss 0.103529 acc 0.781250\n",
      "Validation accuracy: 0.831242\n",
      "Epoch 152 batch 0 loss 0.042377 acc 0.937500\n",
      "Epoch 152 batch 100 loss 0.085143 acc 0.843750\n",
      "Epoch 152 batch 200 loss 0.094676 acc 0.812500\n",
      "Epoch 152 batch 300 loss 0.084392 acc 0.875000\n",
      "Epoch 152 batch 400 loss 0.103040 acc 0.812500\n",
      "Epoch 152 batch 500 loss 0.074454 acc 0.906250\n",
      "Epoch 152 batch 600 loss 0.096468 acc 0.843750\n",
      "Epoch 152 batch 700 loss 0.088059 acc 0.843750\n",
      "Epoch 152 batch 800 loss 0.078230 acc 0.875000\n",
      "Epoch 152 batch 900 loss 0.071991 acc 0.875000\n",
      "Epoch 152 batch 1000 loss 0.080387 acc 0.843750\n",
      "Epoch 152 batch 1100 loss 0.102918 acc 0.781250\n",
      "Validation accuracy: 0.831321\n",
      "Epoch 153 batch 0 loss 0.042306 acc 0.937500\n",
      "Epoch 153 batch 100 loss 0.085356 acc 0.843750\n",
      "Epoch 153 batch 200 loss 0.094697 acc 0.812500\n",
      "Epoch 153 batch 300 loss 0.083829 acc 0.875000\n",
      "Epoch 153 batch 400 loss 0.103035 acc 0.812500\n",
      "Epoch 153 batch 500 loss 0.074295 acc 0.906250\n",
      "Epoch 153 batch 600 loss 0.096530 acc 0.843750\n",
      "Epoch 153 batch 700 loss 0.087983 acc 0.843750\n",
      "Epoch 153 batch 800 loss 0.078237 acc 0.875000\n",
      "Epoch 153 batch 900 loss 0.071944 acc 0.875000\n",
      "Epoch 153 batch 1000 loss 0.080495 acc 0.843750\n",
      "Epoch 153 batch 1100 loss 0.103489 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 154 batch 0 loss 0.043468 acc 0.937500\n",
      "Epoch 154 batch 100 loss 0.085524 acc 0.843750\n",
      "Epoch 154 batch 200 loss 0.094728 acc 0.812500\n",
      "Epoch 154 batch 300 loss 0.081609 acc 0.875000\n",
      "Epoch 154 batch 400 loss 0.103021 acc 0.812500\n",
      "Epoch 154 batch 500 loss 0.074713 acc 0.906250\n",
      "Epoch 154 batch 600 loss 0.096355 acc 0.843750\n",
      "Epoch 154 batch 700 loss 0.088076 acc 0.843750\n",
      "Epoch 154 batch 800 loss 0.078363 acc 0.875000\n",
      "Epoch 154 batch 900 loss 0.071971 acc 0.875000\n",
      "Epoch 154 batch 1000 loss 0.080428 acc 0.843750\n",
      "Epoch 154 batch 1100 loss 0.105027 acc 0.781250\n",
      "Validation accuracy: 0.830137\n",
      "Epoch 155 batch 0 loss 0.041337 acc 0.937500\n",
      "Epoch 155 batch 100 loss 0.085395 acc 0.843750\n",
      "Epoch 155 batch 200 loss 0.094678 acc 0.812500\n",
      "Epoch 155 batch 300 loss 0.084888 acc 0.875000\n",
      "Epoch 155 batch 400 loss 0.102970 acc 0.812500\n",
      "Epoch 155 batch 500 loss 0.074310 acc 0.906250\n",
      "Epoch 155 batch 600 loss 0.096522 acc 0.843750\n",
      "Epoch 155 batch 700 loss 0.088001 acc 0.843750\n",
      "Epoch 155 batch 800 loss 0.078184 acc 0.875000\n",
      "Epoch 155 batch 900 loss 0.071942 acc 0.875000\n",
      "Epoch 155 batch 1000 loss 0.080496 acc 0.843750\n",
      "Epoch 155 batch 1100 loss 0.103455 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.830058\n",
      "Epoch 156 batch 0 loss 0.041148 acc 0.937500\n",
      "Epoch 156 batch 100 loss 0.085296 acc 0.843750\n",
      "Epoch 156 batch 200 loss 0.094655 acc 0.812500\n",
      "Epoch 156 batch 300 loss 0.082568 acc 0.875000\n",
      "Epoch 156 batch 400 loss 0.103052 acc 0.812500\n",
      "Epoch 156 batch 500 loss 0.074688 acc 0.906250\n",
      "Epoch 156 batch 600 loss 0.096331 acc 0.843750\n",
      "Epoch 156 batch 700 loss 0.088089 acc 0.843750\n",
      "Epoch 156 batch 800 loss 0.078233 acc 0.875000\n",
      "Epoch 156 batch 900 loss 0.071942 acc 0.875000\n",
      "Epoch 156 batch 1000 loss 0.080465 acc 0.843750\n",
      "Epoch 156 batch 1100 loss 0.103888 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 157 batch 0 loss 0.040749 acc 0.937500\n",
      "Epoch 157 batch 100 loss 0.085368 acc 0.843750\n",
      "Epoch 157 batch 200 loss 0.094644 acc 0.812500\n",
      "Epoch 157 batch 300 loss 0.083594 acc 0.875000\n",
      "Epoch 157 batch 400 loss 0.103066 acc 0.812500\n",
      "Epoch 157 batch 500 loss 0.074379 acc 0.906250\n",
      "Epoch 157 batch 600 loss 0.096492 acc 0.843750\n",
      "Epoch 157 batch 700 loss 0.087979 acc 0.843750\n",
      "Epoch 157 batch 800 loss 0.078212 acc 0.875000\n",
      "Epoch 157 batch 900 loss 0.071940 acc 0.875000\n",
      "Epoch 157 batch 1000 loss 0.080424 acc 0.843750\n",
      "Epoch 157 batch 1100 loss 0.103559 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 158 batch 0 loss 0.043158 acc 0.937500\n",
      "Epoch 158 batch 100 loss 0.085636 acc 0.812500\n",
      "Epoch 158 batch 200 loss 0.094761 acc 0.812500\n",
      "Epoch 158 batch 300 loss 0.085437 acc 0.875000\n",
      "Epoch 158 batch 400 loss 0.102918 acc 0.812500\n",
      "Epoch 158 batch 500 loss 0.074288 acc 0.906250\n",
      "Epoch 158 batch 600 loss 0.096555 acc 0.843750\n",
      "Epoch 158 batch 700 loss 0.088138 acc 0.843750\n",
      "Epoch 158 batch 800 loss 0.078083 acc 0.875000\n",
      "Epoch 158 batch 900 loss 0.072074 acc 0.875000\n",
      "Epoch 158 batch 1000 loss 0.080371 acc 0.843750\n",
      "Epoch 158 batch 1100 loss 0.103641 acc 0.781250\n",
      "Validation accuracy: 0.831952\n",
      "Epoch 159 batch 0 loss 0.043225 acc 0.937500\n",
      "Epoch 159 batch 100 loss 0.085511 acc 0.843750\n",
      "Epoch 159 batch 200 loss 0.094725 acc 0.812500\n",
      "Epoch 159 batch 300 loss 0.082501 acc 0.875000\n",
      "Epoch 159 batch 400 loss 0.102985 acc 0.812500\n",
      "Epoch 159 batch 500 loss 0.074376 acc 0.906250\n",
      "Epoch 159 batch 600 loss 0.096508 acc 0.843750\n",
      "Epoch 159 batch 700 loss 0.088056 acc 0.843750\n",
      "Epoch 159 batch 800 loss 0.078195 acc 0.875000\n",
      "Epoch 159 batch 900 loss 0.071944 acc 0.875000\n",
      "Epoch 159 batch 1000 loss 0.080486 acc 0.843750\n",
      "Epoch 159 batch 1100 loss 0.104018 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 160 batch 0 loss 0.043626 acc 0.937500\n",
      "Epoch 160 batch 100 loss 0.085554 acc 0.843750\n",
      "Epoch 160 batch 200 loss 0.094701 acc 0.812500\n",
      "Epoch 160 batch 300 loss 0.085247 acc 0.875000\n",
      "Epoch 160 batch 400 loss 0.102876 acc 0.812500\n",
      "Epoch 160 batch 500 loss 0.074246 acc 0.906250\n",
      "Epoch 160 batch 600 loss 0.096548 acc 0.843750\n",
      "Epoch 160 batch 700 loss 0.087991 acc 0.843750\n",
      "Epoch 160 batch 800 loss 0.078256 acc 0.875000\n",
      "Epoch 160 batch 900 loss 0.071967 acc 0.875000\n",
      "Epoch 160 batch 1000 loss 0.080478 acc 0.843750\n",
      "Epoch 160 batch 1100 loss 0.104434 acc 0.781250\n",
      "Validation accuracy: 0.831005\n",
      "Epoch 161 batch 0 loss 0.041654 acc 0.937500\n",
      "Epoch 161 batch 100 loss 0.085537 acc 0.843750\n",
      "Epoch 161 batch 200 loss 0.094704 acc 0.812500\n",
      "Epoch 161 batch 300 loss 0.081533 acc 0.875000\n",
      "Epoch 161 batch 400 loss 0.103343 acc 0.812500\n",
      "Epoch 161 batch 500 loss 0.074079 acc 0.906250\n",
      "Epoch 161 batch 600 loss 0.096612 acc 0.843750\n",
      "Epoch 161 batch 700 loss 0.087912 acc 0.843750\n",
      "Epoch 161 batch 800 loss 0.078170 acc 0.875000\n",
      "Epoch 161 batch 900 loss 0.071948 acc 0.875000\n",
      "Epoch 161 batch 1000 loss 0.080545 acc 0.843750\n",
      "Epoch 161 batch 1100 loss 0.103807 acc 0.781250\n",
      "Validation accuracy: 0.831952\n",
      "Epoch 162 batch 0 loss 0.043168 acc 0.937500\n",
      "Epoch 162 batch 100 loss 0.085562 acc 0.843750\n",
      "Epoch 162 batch 200 loss 0.094722 acc 0.812500\n",
      "Epoch 162 batch 300 loss 0.084748 acc 0.875000\n",
      "Epoch 162 batch 400 loss 0.103038 acc 0.812500\n",
      "Epoch 162 batch 500 loss 0.074079 acc 0.906250\n",
      "Epoch 162 batch 600 loss 0.096621 acc 0.843750\n",
      "Epoch 162 batch 700 loss 0.087932 acc 0.843750\n",
      "Epoch 162 batch 800 loss 0.078152 acc 0.875000\n",
      "Epoch 162 batch 900 loss 0.071972 acc 0.875000\n",
      "Epoch 162 batch 1000 loss 0.080472 acc 0.843750\n",
      "Epoch 162 batch 1100 loss 0.103595 acc 0.781250\n",
      "Validation accuracy: 0.830610\n",
      "Epoch 163 batch 0 loss 0.041426 acc 0.937500\n",
      "Epoch 163 batch 100 loss 0.085420 acc 0.843750\n",
      "Epoch 163 batch 200 loss 0.094685 acc 0.812500\n",
      "Epoch 163 batch 300 loss 0.083603 acc 0.875000\n",
      "Epoch 163 batch 400 loss 0.103056 acc 0.812500\n",
      "Epoch 163 batch 500 loss 0.074266 acc 0.906250\n",
      "Epoch 163 batch 600 loss 0.096548 acc 0.843750\n",
      "Epoch 163 batch 700 loss 0.088010 acc 0.843750\n",
      "Epoch 163 batch 800 loss 0.078210 acc 0.875000\n",
      "Epoch 163 batch 900 loss 0.071975 acc 0.875000\n",
      "Epoch 163 batch 1000 loss 0.080437 acc 0.843750\n",
      "Epoch 163 batch 1100 loss 0.103568 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 164 batch 0 loss 0.043094 acc 0.937500\n",
      "Epoch 164 batch 100 loss 0.085614 acc 0.843750\n",
      "Epoch 164 batch 200 loss 0.094751 acc 0.812500\n",
      "Epoch 164 batch 300 loss 0.081618 acc 0.875000\n",
      "Epoch 164 batch 400 loss 0.103128 acc 0.812500\n",
      "Epoch 164 batch 500 loss 0.074605 acc 0.906250\n",
      "Epoch 164 batch 600 loss 0.096406 acc 0.843750\n",
      "Epoch 164 batch 700 loss 0.088099 acc 0.843750\n",
      "Epoch 164 batch 800 loss 0.078175 acc 0.875000\n",
      "Epoch 164 batch 900 loss 0.071948 acc 0.875000\n",
      "Epoch 164 batch 1000 loss 0.080343 acc 0.843750\n",
      "Epoch 164 batch 1100 loss 0.106708 acc 0.781250\n",
      "Validation accuracy: 0.831005\n",
      "Epoch 165 batch 0 loss 0.041939 acc 0.937500\n",
      "Epoch 165 batch 100 loss 0.085673 acc 0.812500\n",
      "Epoch 165 batch 200 loss 0.094758 acc 0.812500\n",
      "Epoch 165 batch 300 loss 0.083489 acc 0.875000\n",
      "Epoch 165 batch 400 loss 0.102875 acc 0.812500\n",
      "Epoch 165 batch 500 loss 0.074420 acc 0.906250\n",
      "Epoch 165 batch 600 loss 0.096433 acc 0.843750\n",
      "Epoch 165 batch 700 loss 0.088093 acc 0.843750\n",
      "Epoch 165 batch 800 loss 0.078133 acc 0.875000\n",
      "Epoch 165 batch 900 loss 0.071982 acc 0.875000\n",
      "Epoch 165 batch 1000 loss 0.080295 acc 0.843750\n",
      "Epoch 165 batch 1100 loss 0.106670 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 166 batch 0 loss 0.043273 acc 0.937500\n",
      "Epoch 166 batch 100 loss 0.085480 acc 0.843750\n",
      "Epoch 166 batch 200 loss 0.094767 acc 0.812500\n",
      "Epoch 166 batch 300 loss 0.082488 acc 0.875000\n",
      "Epoch 166 batch 400 loss 0.102879 acc 0.812500\n",
      "Epoch 166 batch 500 loss 0.074283 acc 0.906250\n",
      "Epoch 166 batch 600 loss 0.096562 acc 0.843750\n",
      "Epoch 166 batch 700 loss 0.088150 acc 0.843750\n",
      "Epoch 166 batch 800 loss 0.078114 acc 0.875000\n",
      "Epoch 166 batch 900 loss 0.071975 acc 0.875000\n",
      "Epoch 166 batch 1000 loss 0.080457 acc 0.843750\n",
      "Epoch 166 batch 1100 loss 0.103133 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 167 batch 0 loss 0.043262 acc 0.937500\n",
      "Epoch 167 batch 100 loss 0.085600 acc 0.843750\n",
      "Epoch 167 batch 200 loss 0.094774 acc 0.812500\n",
      "Epoch 167 batch 300 loss 0.083949 acc 0.875000\n",
      "Epoch 167 batch 400 loss 0.103294 acc 0.812500\n",
      "Epoch 167 batch 500 loss 0.073676 acc 0.906250\n",
      "Epoch 167 batch 600 loss 0.096805 acc 0.843750\n",
      "Epoch 167 batch 700 loss 0.087972 acc 0.843750\n",
      "Epoch 167 batch 800 loss 0.078101 acc 0.875000\n",
      "Epoch 167 batch 900 loss 0.071947 acc 0.875000\n",
      "Epoch 167 batch 1000 loss 0.080408 acc 0.843750\n",
      "Epoch 167 batch 1100 loss 0.103465 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 168 batch 0 loss 0.043063 acc 0.937500\n",
      "Epoch 168 batch 100 loss 0.085295 acc 0.843750\n",
      "Epoch 168 batch 200 loss 0.094738 acc 0.812500\n",
      "Epoch 168 batch 300 loss 0.085393 acc 0.875000\n",
      "Epoch 168 batch 400 loss 0.102976 acc 0.812500\n",
      "Epoch 168 batch 500 loss 0.074052 acc 0.906250\n",
      "Epoch 168 batch 600 loss 0.096536 acc 0.843750\n",
      "Epoch 168 batch 700 loss 0.088156 acc 0.843750\n",
      "Epoch 168 batch 800 loss 0.078118 acc 0.875000\n",
      "Epoch 168 batch 900 loss 0.071997 acc 0.875000\n",
      "Epoch 168 batch 1000 loss 0.080378 acc 0.843750\n",
      "Epoch 168 batch 1100 loss 0.103571 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 169 batch 0 loss 0.043132 acc 0.937500\n",
      "Epoch 169 batch 100 loss 0.085588 acc 0.843750\n",
      "Epoch 169 batch 200 loss 0.094785 acc 0.812500\n",
      "Epoch 169 batch 300 loss 0.083342 acc 0.875000\n",
      "Epoch 169 batch 400 loss 0.103084 acc 0.812500\n",
      "Epoch 169 batch 500 loss 0.074269 acc 0.906250\n",
      "Epoch 169 batch 600 loss 0.096530 acc 0.843750\n",
      "Epoch 169 batch 700 loss 0.088077 acc 0.843750\n",
      "Epoch 169 batch 800 loss 0.078107 acc 0.875000\n",
      "Epoch 169 batch 900 loss 0.071997 acc 0.875000\n",
      "Epoch 169 batch 1000 loss 0.080280 acc 0.843750\n",
      "Epoch 169 batch 1100 loss 0.106887 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.831873\n",
      "Epoch 170 batch 0 loss 0.044024 acc 0.937500\n",
      "Epoch 170 batch 100 loss 0.085659 acc 0.812500\n",
      "Epoch 170 batch 200 loss 0.094798 acc 0.812500\n",
      "Epoch 170 batch 300 loss 0.084185 acc 0.875000\n",
      "Epoch 170 batch 400 loss 0.102909 acc 0.812500\n",
      "Epoch 170 batch 500 loss 0.073454 acc 0.906250\n",
      "Epoch 170 batch 600 loss 0.096695 acc 0.843750\n",
      "Epoch 170 batch 700 loss 0.088167 acc 0.843750\n",
      "Epoch 170 batch 800 loss 0.078032 acc 0.875000\n",
      "Epoch 170 batch 900 loss 0.072053 acc 0.875000\n",
      "Epoch 170 batch 1000 loss 0.080319 acc 0.843750\n",
      "Epoch 170 batch 1100 loss 0.104624 acc 0.781250\n",
      "Validation accuracy: 0.830610\n",
      "Epoch 171 batch 0 loss 0.041253 acc 0.937500\n",
      "Epoch 171 batch 100 loss 0.085421 acc 0.843750\n",
      "Epoch 171 batch 200 loss 0.094690 acc 0.812500\n",
      "Epoch 171 batch 300 loss 0.084859 acc 0.875000\n",
      "Epoch 171 batch 400 loss 0.103017 acc 0.812500\n",
      "Epoch 171 batch 500 loss 0.073943 acc 0.906250\n",
      "Epoch 171 batch 600 loss 0.096676 acc 0.843750\n",
      "Epoch 171 batch 700 loss 0.088010 acc 0.843750\n",
      "Epoch 171 batch 800 loss 0.078106 acc 0.875000\n",
      "Epoch 171 batch 900 loss 0.071992 acc 0.875000\n",
      "Epoch 171 batch 1000 loss 0.080447 acc 0.843750\n",
      "Epoch 171 batch 1100 loss 0.103200 acc 0.781250\n",
      "Validation accuracy: 0.831873\n",
      "Epoch 172 batch 0 loss 0.044115 acc 0.937500\n",
      "Epoch 172 batch 100 loss 0.085417 acc 0.843750\n",
      "Epoch 172 batch 200 loss 0.094774 acc 0.812500\n",
      "Epoch 172 batch 300 loss 0.084268 acc 0.875000\n",
      "Epoch 172 batch 400 loss 0.103103 acc 0.812500\n",
      "Epoch 172 batch 500 loss 0.073985 acc 0.906250\n",
      "Epoch 172 batch 600 loss 0.096666 acc 0.843750\n",
      "Epoch 172 batch 700 loss 0.088013 acc 0.843750\n",
      "Epoch 172 batch 800 loss 0.078051 acc 0.875000\n",
      "Epoch 172 batch 900 loss 0.072014 acc 0.875000\n",
      "Epoch 172 batch 1000 loss 0.080343 acc 0.843750\n",
      "Epoch 172 batch 1100 loss 0.103652 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 173 batch 0 loss 0.040921 acc 0.937500\n",
      "Epoch 173 batch 100 loss 0.085586 acc 0.843750\n",
      "Epoch 173 batch 200 loss 0.094699 acc 0.812500\n",
      "Epoch 173 batch 300 loss 0.085049 acc 0.875000\n",
      "Epoch 173 batch 400 loss 0.102992 acc 0.812500\n",
      "Epoch 173 batch 500 loss 0.074012 acc 0.906250\n",
      "Epoch 173 batch 600 loss 0.096640 acc 0.843750\n",
      "Epoch 173 batch 700 loss 0.087990 acc 0.843750\n",
      "Epoch 173 batch 800 loss 0.078125 acc 0.875000\n",
      "Epoch 173 batch 900 loss 0.071990 acc 0.875000\n",
      "Epoch 173 batch 1000 loss 0.080454 acc 0.843750\n",
      "Epoch 173 batch 1100 loss 0.103484 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 174 batch 0 loss 0.044344 acc 0.937500\n",
      "Epoch 174 batch 100 loss 0.085596 acc 0.812500\n",
      "Epoch 174 batch 200 loss 0.094797 acc 0.812500\n",
      "Epoch 174 batch 300 loss 0.082899 acc 0.875000\n",
      "Epoch 174 batch 400 loss 0.102962 acc 0.812500\n",
      "Epoch 174 batch 500 loss 0.073799 acc 0.906250\n",
      "Epoch 174 batch 600 loss 0.096651 acc 0.843750\n",
      "Epoch 174 batch 700 loss 0.088112 acc 0.843750\n",
      "Epoch 174 batch 800 loss 0.078080 acc 0.875000\n",
      "Epoch 174 batch 900 loss 0.071997 acc 0.875000\n",
      "Epoch 174 batch 1000 loss 0.080335 acc 0.843750\n",
      "Epoch 174 batch 1100 loss 0.105050 acc 0.781250\n",
      "Validation accuracy: 0.832110\n",
      "Epoch 175 batch 0 loss 0.043463 acc 0.937500\n",
      "Epoch 175 batch 100 loss 0.085634 acc 0.843750\n",
      "Epoch 175 batch 200 loss 0.094741 acc 0.812500\n",
      "Epoch 175 batch 300 loss 0.084968 acc 0.875000\n",
      "Epoch 175 batch 400 loss 0.103209 acc 0.812500\n",
      "Epoch 175 batch 500 loss 0.073932 acc 0.906250\n",
      "Epoch 175 batch 600 loss 0.096663 acc 0.843750\n",
      "Epoch 175 batch 700 loss 0.087972 acc 0.843750\n",
      "Epoch 175 batch 800 loss 0.078088 acc 0.875000\n",
      "Epoch 175 batch 900 loss 0.071965 acc 0.875000\n",
      "Epoch 175 batch 1000 loss 0.080464 acc 0.843750\n",
      "Epoch 175 batch 1100 loss 0.103331 acc 0.781250\n",
      "Validation accuracy: 0.831952\n",
      "Epoch 176 batch 0 loss 0.043422 acc 0.937500\n",
      "Epoch 176 batch 100 loss 0.085736 acc 0.812500\n",
      "Epoch 176 batch 200 loss 0.094819 acc 0.812500\n",
      "Epoch 176 batch 300 loss 0.084498 acc 0.875000\n",
      "Epoch 176 batch 400 loss 0.102986 acc 0.812500\n",
      "Epoch 176 batch 500 loss 0.073864 acc 0.906250\n",
      "Epoch 176 batch 600 loss 0.096676 acc 0.843750\n",
      "Epoch 176 batch 700 loss 0.088121 acc 0.843750\n",
      "Epoch 176 batch 800 loss 0.078271 acc 0.875000\n",
      "Epoch 176 batch 900 loss 0.072001 acc 0.875000\n",
      "Epoch 176 batch 1000 loss 0.080505 acc 0.843750\n",
      "Epoch 176 batch 1100 loss 0.107251 acc 0.781250\n",
      "Validation accuracy: 0.832268\n",
      "Epoch 177 batch 0 loss 0.044552 acc 0.937500\n",
      "Epoch 177 batch 100 loss 0.085590 acc 0.812500\n",
      "Epoch 177 batch 200 loss 0.094811 acc 0.812500\n",
      "Epoch 177 batch 300 loss 0.081533 acc 0.875000\n",
      "Epoch 177 batch 400 loss 0.103247 acc 0.812500\n",
      "Epoch 177 batch 500 loss 0.074220 acc 0.906250\n",
      "Epoch 177 batch 600 loss 0.096507 acc 0.843750\n",
      "Epoch 177 batch 700 loss 0.088212 acc 0.843750\n",
      "Epoch 177 batch 800 loss 0.077945 acc 0.875000\n",
      "Epoch 177 batch 900 loss 0.072035 acc 0.875000\n",
      "Epoch 177 batch 1000 loss 0.080337 acc 0.843750\n",
      "Epoch 177 batch 1100 loss 0.103794 acc 0.781250\n",
      "Validation accuracy: 0.830610\n",
      "Epoch 178 batch 0 loss 0.041360 acc 0.937500\n",
      "Epoch 178 batch 100 loss 0.085402 acc 0.843750\n",
      "Epoch 178 batch 200 loss 0.094706 acc 0.812500\n",
      "Epoch 178 batch 300 loss 0.083385 acc 0.875000\n",
      "Epoch 178 batch 400 loss 0.103098 acc 0.812500\n",
      "Epoch 178 batch 500 loss 0.073732 acc 0.906250\n",
      "Epoch 178 batch 600 loss 0.096497 acc 0.843750\n",
      "Epoch 178 batch 700 loss 0.088195 acc 0.843750\n",
      "Epoch 178 batch 800 loss 0.078068 acc 0.875000\n",
      "Epoch 178 batch 900 loss 0.071989 acc 0.875000\n",
      "Epoch 178 batch 1000 loss 0.080338 acc 0.843750\n",
      "Epoch 178 batch 1100 loss 0.105880 acc 0.781250\n",
      "Validation accuracy: 0.830531\n",
      "Epoch 179 batch 0 loss 0.041494 acc 0.937500\n",
      "Epoch 179 batch 100 loss 0.085634 acc 0.812500\n",
      "Epoch 179 batch 200 loss 0.094756 acc 0.812500\n",
      "Epoch 179 batch 300 loss 0.083761 acc 0.875000\n",
      "Epoch 179 batch 400 loss 0.103182 acc 0.812500\n",
      "Epoch 179 batch 500 loss 0.073943 acc 0.906250\n",
      "Epoch 179 batch 600 loss 0.096673 acc 0.843750\n",
      "Epoch 179 batch 700 loss 0.087948 acc 0.843750\n",
      "Epoch 179 batch 800 loss 0.078062 acc 0.875000\n",
      "Epoch 179 batch 900 loss 0.071968 acc 0.875000\n",
      "Epoch 179 batch 1000 loss 0.080361 acc 0.843750\n",
      "Epoch 179 batch 1100 loss 0.103428 acc 0.781250\n",
      "Validation accuracy: 0.831321\n",
      "Epoch 180 batch 0 loss 0.042272 acc 0.937500\n",
      "Epoch 180 batch 100 loss 0.085626 acc 0.812500\n",
      "Epoch 180 batch 200 loss 0.094773 acc 0.812500\n",
      "Epoch 180 batch 300 loss 0.081531 acc 0.875000\n",
      "Epoch 180 batch 400 loss 0.103075 acc 0.812500\n",
      "Epoch 180 batch 500 loss 0.074259 acc 0.906250\n",
      "Epoch 180 batch 600 loss 0.096479 acc 0.843750\n",
      "Epoch 180 batch 700 loss 0.088075 acc 0.843750\n",
      "Epoch 180 batch 800 loss 0.078118 acc 0.875000\n",
      "Epoch 180 batch 900 loss 0.071956 acc 0.875000\n",
      "Epoch 180 batch 1000 loss 0.080431 acc 0.843750\n",
      "Epoch 180 batch 1100 loss 0.104033 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 181 batch 0 loss 0.040806 acc 0.937500\n",
      "Epoch 181 batch 100 loss 0.085594 acc 0.843750\n",
      "Epoch 181 batch 200 loss 0.094708 acc 0.812500\n",
      "Epoch 181 batch 300 loss 0.083532 acc 0.875000\n",
      "Epoch 181 batch 400 loss 0.103200 acc 0.812500\n",
      "Epoch 181 batch 500 loss 0.074038 acc 0.906250\n",
      "Epoch 181 batch 600 loss 0.096597 acc 0.843750\n",
      "Epoch 181 batch 700 loss 0.087952 acc 0.843750\n",
      "Epoch 181 batch 800 loss 0.078096 acc 0.875000\n",
      "Epoch 181 batch 900 loss 0.071951 acc 0.875000\n",
      "Epoch 181 batch 1000 loss 0.080396 acc 0.843750\n",
      "Epoch 181 batch 1100 loss 0.103682 acc 0.781250\n",
      "Validation accuracy: 0.832031\n",
      "Epoch 182 batch 0 loss 0.043568 acc 0.937500\n",
      "Epoch 182 batch 100 loss 0.085804 acc 0.812500\n",
      "Epoch 182 batch 200 loss 0.094825 acc 0.812500\n",
      "Epoch 182 batch 300 loss 0.083573 acc 0.875000\n",
      "Epoch 182 batch 400 loss 0.102917 acc 0.812500\n",
      "Epoch 182 batch 500 loss 0.073942 acc 0.906250\n",
      "Epoch 182 batch 600 loss 0.096608 acc 0.843750\n",
      "Epoch 182 batch 700 loss 0.088091 acc 0.843750\n",
      "Epoch 182 batch 800 loss 0.078372 acc 0.875000\n",
      "Epoch 182 batch 900 loss 0.071986 acc 0.875000\n",
      "Epoch 182 batch 1000 loss 0.080464 acc 0.843750\n",
      "Epoch 182 batch 1100 loss 0.103664 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 183 batch 0 loss 0.042907 acc 0.937500\n",
      "Epoch 183 batch 100 loss 0.085743 acc 0.812500\n",
      "Epoch 183 batch 200 loss 0.094807 acc 0.812500\n",
      "Epoch 183 batch 300 loss 0.081427 acc 0.875000\n",
      "Epoch 183 batch 400 loss 0.103095 acc 0.812500\n",
      "Epoch 183 batch 500 loss 0.073741 acc 0.906250\n",
      "Epoch 183 batch 600 loss 0.096520 acc 0.843750\n",
      "Epoch 183 batch 700 loss 0.088225 acc 0.843750\n",
      "Epoch 183 batch 800 loss 0.078019 acc 0.875000\n",
      "Epoch 183 batch 900 loss 0.071982 acc 0.875000\n",
      "Epoch 183 batch 1000 loss 0.080294 acc 0.843750\n",
      "Epoch 183 batch 1100 loss 0.105096 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.829900\n",
      "Epoch 184 batch 0 loss 0.040595 acc 0.937500\n",
      "Epoch 184 batch 100 loss 0.085670 acc 0.812500\n",
      "Epoch 184 batch 200 loss 0.094718 acc 0.812500\n",
      "Epoch 184 batch 300 loss 0.085922 acc 0.875000\n",
      "Epoch 184 batch 400 loss 0.102924 acc 0.812500\n",
      "Epoch 184 batch 500 loss 0.073921 acc 0.906250\n",
      "Epoch 184 batch 600 loss 0.096551 acc 0.843750\n",
      "Epoch 184 batch 700 loss 0.088062 acc 0.843750\n",
      "Epoch 184 batch 800 loss 0.078056 acc 0.875000\n",
      "Epoch 184 batch 900 loss 0.071962 acc 0.875000\n",
      "Epoch 184 batch 1000 loss 0.080448 acc 0.843750\n",
      "Epoch 184 batch 1100 loss 0.103998 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 185 batch 0 loss 0.043111 acc 0.937500\n",
      "Epoch 185 batch 100 loss 0.085720 acc 0.812500\n",
      "Epoch 185 batch 200 loss 0.094824 acc 0.812500\n",
      "Epoch 185 batch 300 loss 0.081747 acc 0.875000\n",
      "Epoch 185 batch 400 loss 0.103132 acc 0.812500\n",
      "Epoch 185 batch 500 loss 0.074171 acc 0.906250\n",
      "Epoch 185 batch 600 loss 0.096538 acc 0.843750\n",
      "Epoch 185 batch 700 loss 0.088135 acc 0.843750\n",
      "Epoch 185 batch 800 loss 0.078196 acc 0.875000\n",
      "Epoch 185 batch 900 loss 0.071976 acc 0.875000\n",
      "Epoch 185 batch 1000 loss 0.080453 acc 0.843750\n",
      "Epoch 185 batch 1100 loss 0.103317 acc 0.781250\n",
      "Validation accuracy: 0.830058\n",
      "Epoch 186 batch 0 loss 0.040575 acc 0.937500\n",
      "Epoch 186 batch 100 loss 0.085597 acc 0.843750\n",
      "Epoch 186 batch 200 loss 0.094772 acc 0.812500\n",
      "Epoch 186 batch 300 loss 0.081427 acc 0.875000\n",
      "Epoch 186 batch 400 loss 0.103042 acc 0.812500\n",
      "Epoch 186 batch 500 loss 0.074336 acc 0.906250\n",
      "Epoch 186 batch 600 loss 0.096431 acc 0.843750\n",
      "Epoch 186 batch 700 loss 0.088139 acc 0.843750\n",
      "Epoch 186 batch 800 loss 0.078088 acc 0.875000\n",
      "Epoch 186 batch 900 loss 0.072029 acc 0.875000\n",
      "Epoch 186 batch 1000 loss 0.080253 acc 0.843750\n",
      "Epoch 186 batch 1100 loss 0.105938 acc 0.781250\n",
      "Validation accuracy: 0.830452\n",
      "Epoch 187 batch 0 loss 0.041294 acc 0.937500\n",
      "Epoch 187 batch 100 loss 0.085490 acc 0.843750\n",
      "Epoch 187 batch 200 loss 0.094727 acc 0.812500\n",
      "Epoch 187 batch 300 loss 0.084831 acc 0.875000\n",
      "Epoch 187 batch 400 loss 0.102813 acc 0.812500\n",
      "Epoch 187 batch 500 loss 0.073782 acc 0.906250\n",
      "Epoch 187 batch 600 loss 0.096586 acc 0.843750\n",
      "Epoch 187 batch 700 loss 0.088104 acc 0.843750\n",
      "Epoch 187 batch 800 loss 0.078026 acc 0.875000\n",
      "Epoch 187 batch 900 loss 0.071988 acc 0.875000\n",
      "Epoch 187 batch 1000 loss 0.080418 acc 0.843750\n",
      "Epoch 187 batch 1100 loss 0.103757 acc 0.781250\n",
      "Validation accuracy: 0.830689\n",
      "Epoch 188 batch 0 loss 0.041809 acc 0.937500\n",
      "Epoch 188 batch 100 loss 0.085507 acc 0.843750\n",
      "Epoch 188 batch 200 loss 0.094785 acc 0.812500\n",
      "Epoch 188 batch 300 loss 0.081460 acc 0.875000\n",
      "Epoch 188 batch 400 loss 0.103192 acc 0.812500\n",
      "Epoch 188 batch 500 loss 0.074106 acc 0.906250\n",
      "Epoch 188 batch 600 loss 0.096519 acc 0.843750\n",
      "Epoch 188 batch 700 loss 0.088124 acc 0.843750\n",
      "Epoch 188 batch 800 loss 0.078076 acc 0.875000\n",
      "Epoch 188 batch 900 loss 0.072050 acc 0.875000\n",
      "Epoch 188 batch 1000 loss 0.080320 acc 0.843750\n",
      "Epoch 188 batch 1100 loss 0.102958 acc 0.781250\n",
      "Validation accuracy: 0.830215\n",
      "Epoch 189 batch 0 loss 0.041416 acc 0.937500\n",
      "Epoch 189 batch 100 loss 0.085445 acc 0.843750\n",
      "Epoch 189 batch 200 loss 0.094783 acc 0.812500\n",
      "Epoch 189 batch 300 loss 0.084273 acc 0.875000\n",
      "Epoch 189 batch 400 loss 0.103015 acc 0.812500\n",
      "Epoch 189 batch 500 loss 0.074735 acc 0.906250\n",
      "Epoch 189 batch 600 loss 0.096296 acc 0.843750\n",
      "Epoch 189 batch 700 loss 0.088222 acc 0.843750\n",
      "Epoch 189 batch 800 loss 0.078042 acc 0.875000\n",
      "Epoch 189 batch 900 loss 0.071979 acc 0.875000\n",
      "Epoch 189 batch 1000 loss 0.080302 acc 0.843750\n",
      "Epoch 189 batch 1100 loss 0.103854 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 190 batch 0 loss 0.040825 acc 0.937500\n",
      "Epoch 190 batch 100 loss 0.085429 acc 0.843750\n",
      "Epoch 190 batch 200 loss 0.094766 acc 0.812500\n",
      "Epoch 190 batch 300 loss 0.081455 acc 0.875000\n",
      "Epoch 190 batch 400 loss 0.103272 acc 0.812500\n",
      "Epoch 190 batch 500 loss 0.073898 acc 0.906250\n",
      "Epoch 190 batch 600 loss 0.096612 acc 0.843750\n",
      "Epoch 190 batch 700 loss 0.087991 acc 0.843750\n",
      "Epoch 190 batch 800 loss 0.078120 acc 0.875000\n",
      "Epoch 190 batch 900 loss 0.071958 acc 0.875000\n",
      "Epoch 190 batch 1000 loss 0.080380 acc 0.843750\n",
      "Epoch 190 batch 1100 loss 0.103626 acc 0.781250\n",
      "Validation accuracy: 0.830373\n",
      "Epoch 191 batch 0 loss 0.041027 acc 0.937500\n",
      "Epoch 191 batch 100 loss 0.085378 acc 0.843750\n",
      "Epoch 191 batch 200 loss 0.094727 acc 0.812500\n",
      "Epoch 191 batch 300 loss 0.081456 acc 0.875000\n",
      "Epoch 191 batch 400 loss 0.103051 acc 0.812500\n",
      "Epoch 191 batch 500 loss 0.074545 acc 0.906250\n",
      "Epoch 191 batch 600 loss 0.096315 acc 0.843750\n",
      "Epoch 191 batch 700 loss 0.088177 acc 0.843750\n",
      "Epoch 191 batch 800 loss 0.078088 acc 0.875000\n",
      "Epoch 191 batch 900 loss 0.071961 acc 0.875000\n",
      "Epoch 191 batch 1000 loss 0.080441 acc 0.843750\n",
      "Epoch 191 batch 1100 loss 0.103253 acc 0.781250\n",
      "Validation accuracy: 0.832189\n",
      "Epoch 192 batch 0 loss 0.044064 acc 0.937500\n",
      "Epoch 192 batch 100 loss 0.085801 acc 0.812500\n",
      "Epoch 192 batch 200 loss 0.094845 acc 0.812500\n",
      "Epoch 192 batch 300 loss 0.082882 acc 0.875000\n",
      "Epoch 192 batch 400 loss 0.102916 acc 0.812500\n",
      "Epoch 192 batch 500 loss 0.073813 acc 0.906250\n",
      "Epoch 192 batch 600 loss 0.096646 acc 0.843750\n",
      "Epoch 192 batch 700 loss 0.088298 acc 0.843750\n",
      "Epoch 192 batch 800 loss 0.077951 acc 0.875000\n",
      "Epoch 192 batch 900 loss 0.071981 acc 0.875000\n",
      "Epoch 192 batch 1000 loss 0.080425 acc 0.843750\n",
      "Epoch 192 batch 1100 loss 0.103621 acc 0.781250\n",
      "Validation accuracy: 0.831005\n",
      "Epoch 193 batch 0 loss 0.041767 acc 0.937500\n",
      "Epoch 193 batch 100 loss 0.085580 acc 0.843750\n",
      "Epoch 193 batch 200 loss 0.094770 acc 0.812500\n",
      "Epoch 193 batch 300 loss 0.081486 acc 0.875000\n",
      "Epoch 193 batch 400 loss 0.103140 acc 0.812500\n",
      "Epoch 193 batch 500 loss 0.074160 acc 0.906250\n",
      "Epoch 193 batch 600 loss 0.096513 acc 0.843750\n",
      "Epoch 193 batch 700 loss 0.088084 acc 0.843750\n",
      "Epoch 193 batch 800 loss 0.078057 acc 0.875000\n",
      "Epoch 193 batch 900 loss 0.071944 acc 0.875000\n",
      "Epoch 193 batch 1000 loss 0.080417 acc 0.843750\n",
      "Epoch 193 batch 1100 loss 0.104134 acc 0.781250\n",
      "Validation accuracy: 0.829979\n",
      "Epoch 194 batch 0 loss 0.040744 acc 0.937500\n",
      "Epoch 194 batch 100 loss 0.085942 acc 0.812500\n",
      "Epoch 194 batch 200 loss 0.094734 acc 0.812500\n",
      "Epoch 194 batch 300 loss 0.081659 acc 0.875000\n",
      "Epoch 194 batch 400 loss 0.103055 acc 0.812500\n",
      "Epoch 194 batch 500 loss 0.074479 acc 0.906250\n",
      "Epoch 194 batch 600 loss 0.096345 acc 0.843750\n",
      "Epoch 194 batch 700 loss 0.088070 acc 0.843750\n",
      "Epoch 194 batch 800 loss 0.078068 acc 0.875000\n",
      "Epoch 194 batch 900 loss 0.071945 acc 0.875000\n",
      "Epoch 194 batch 1000 loss 0.080428 acc 0.843750\n",
      "Epoch 194 batch 1100 loss 0.103502 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 195 batch 0 loss 0.040845 acc 0.937500\n",
      "Epoch 195 batch 100 loss 0.085533 acc 0.843750\n",
      "Epoch 195 batch 200 loss 0.094739 acc 0.812500\n",
      "Epoch 195 batch 300 loss 0.083936 acc 0.875000\n",
      "Epoch 195 batch 400 loss 0.103085 acc 0.812500\n",
      "Epoch 195 batch 500 loss 0.074211 acc 0.906250\n",
      "Epoch 195 batch 600 loss 0.096464 acc 0.843750\n",
      "Epoch 195 batch 700 loss 0.088034 acc 0.843750\n",
      "Epoch 195 batch 800 loss 0.078036 acc 0.875000\n",
      "Epoch 195 batch 900 loss 0.071949 acc 0.875000\n",
      "Epoch 195 batch 1000 loss 0.080402 acc 0.843750\n",
      "Epoch 195 batch 1100 loss 0.103750 acc 0.781250\n",
      "Validation accuracy: 0.829900\n",
      "Epoch 196 batch 0 loss 0.040916 acc 0.937500\n",
      "Epoch 196 batch 100 loss 0.085621 acc 0.812500\n",
      "Epoch 196 batch 200 loss 0.094729 acc 0.812500\n",
      "Epoch 196 batch 300 loss 0.084404 acc 0.875000\n",
      "Epoch 196 batch 400 loss 0.103034 acc 0.812500\n",
      "Epoch 196 batch 500 loss 0.074128 acc 0.906250\n",
      "Epoch 196 batch 600 loss 0.096491 acc 0.843750\n",
      "Epoch 196 batch 700 loss 0.087996 acc 0.843750\n",
      "Epoch 196 batch 800 loss 0.078018 acc 0.875000\n",
      "Epoch 196 batch 900 loss 0.071953 acc 0.875000\n",
      "Epoch 196 batch 1000 loss 0.080401 acc 0.843750\n",
      "Epoch 196 batch 1100 loss 0.104005 acc 0.781250\n",
      "Validation accuracy: 0.831794\n",
      "Epoch 197 batch 0 loss 0.042901 acc 0.937500\n",
      "Epoch 197 batch 100 loss 0.085682 acc 0.812500\n",
      "Epoch 197 batch 200 loss 0.094804 acc 0.812500\n",
      "Epoch 197 batch 300 loss 0.081453 acc 0.875000\n",
      "Epoch 197 batch 400 loss 0.103155 acc 0.812500\n",
      "Epoch 197 batch 500 loss 0.074372 acc 0.906250\n",
      "Epoch 197 batch 600 loss 0.096409 acc 0.843750\n",
      "Epoch 197 batch 700 loss 0.088064 acc 0.843750\n",
      "Epoch 197 batch 800 loss 0.078023 acc 0.875000\n",
      "Epoch 197 batch 900 loss 0.072009 acc 0.875000\n",
      "Epoch 197 batch 1000 loss 0.080243 acc 0.843750\n",
      "Epoch 197 batch 1100 loss 0.105476 acc 0.781250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.831873\n",
      "Epoch 198 batch 0 loss 0.042867 acc 0.937500\n",
      "Epoch 198 batch 100 loss 0.085954 acc 0.812500\n",
      "Epoch 198 batch 200 loss 0.094810 acc 0.812500\n",
      "Epoch 198 batch 300 loss 0.085854 acc 0.875000\n",
      "Epoch 198 batch 400 loss 0.102938 acc 0.812500\n",
      "Epoch 198 batch 500 loss 0.073476 acc 0.906250\n",
      "Epoch 198 batch 600 loss 0.096648 acc 0.843750\n",
      "Epoch 198 batch 700 loss 0.088038 acc 0.843750\n",
      "Epoch 198 batch 800 loss 0.077934 acc 0.875000\n",
      "Epoch 198 batch 900 loss 0.071960 acc 0.875000\n",
      "Epoch 198 batch 1000 loss 0.080352 acc 0.843750\n",
      "Epoch 198 batch 1100 loss 0.106940 acc 0.781250\n",
      "Validation accuracy: 0.830847\n",
      "Epoch 199 batch 0 loss 0.041741 acc 0.937500\n",
      "Epoch 199 batch 100 loss 0.085617 acc 0.812500\n",
      "Epoch 199 batch 200 loss 0.094810 acc 0.812500\n",
      "Epoch 199 batch 300 loss 0.083087 acc 0.875000\n",
      "Epoch 199 batch 400 loss 0.103224 acc 0.812500\n",
      "Epoch 199 batch 500 loss 0.073901 acc 0.906250\n",
      "Epoch 199 batch 600 loss 0.096609 acc 0.843750\n",
      "Epoch 199 batch 700 loss 0.088030 acc 0.843750\n",
      "Epoch 199 batch 800 loss 0.077973 acc 0.875000\n",
      "Epoch 199 batch 900 loss 0.071979 acc 0.875000\n",
      "Epoch 199 batch 1000 loss 0.080257 acc 0.843750\n",
      "Epoch 199 batch 1100 loss 0.106681 acc 0.781250\n",
      "Validation accuracy: 0.831005\n",
      "Accuracy:  tensor(83.3768, dtype=torch.float64)\n",
      "Loss:  tensor(0.0004, grad_fn=<DivBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Setup input and train protoNN\n",
    "protoNN = ProtoNN(dataDimension, PROJECTION_DIM,\n",
    "                  NUM_PROTOTYPES, numClasses,\n",
    "                  gamma, W=W, B=B)\n",
    "\n",
    "trainer = ProtoNNTrainer(protoNN, REG_W, REG_B, REG_Z,\n",
    "                         SPAR_W, SPAR_B, SPAR_Z,\n",
    "                         LEARNING_RATE, lossType='l2')\n",
    "\n",
    "trainer.train(BATCH_SIZE, NUM_EPOCHS, X_train, X_test, y_train, y_test,\n",
    "              printStep=100, valStep=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e9165dae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model size constraint (Bytes):  2180\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def countnnZ(A, s, bytesPerVar=4):\n",
    "    '''\n",
    "    Returns # of non-zeros and representative size of the tensor\n",
    "    Uses dense for s >= 0.5 - 4 byte\n",
    "    Else uses sparse - 8 byte\n",
    "    '''\n",
    "    params = 1\n",
    "    hasSparse = False\n",
    "    for i in range(0, len(A.shape)):\n",
    "        params *= int(A.shape[i])\n",
    "    if s < 0.5:\n",
    "        nnZ = np.ceil(params * s)\n",
    "        hasSparse = True\n",
    "        return nnZ, nnZ * 2 * bytesPerVar, hasSparse\n",
    "    else:\n",
    "        nnZ = params\n",
    "        return nnZ, nnZ * bytesPerVar, hasSparse\n",
    "    \n",
    "def getModelSize(matrixList, sparcityList, expected=True, bytesPerVar=4):\n",
    "    '''\n",
    "    expected: Expected size according to the parameters set. The number of\n",
    "        zeros could actually be more than that is required to satisfy the\n",
    "        sparsity constraint.\n",
    "    '''\n",
    "    nnzList, sizeList, isSparseList = [], [], []\n",
    "    hasSparse = False\n",
    "    for i in range(len(matrixList)):\n",
    "        A, s = matrixList[i], sparcityList[i]\n",
    "        assert A.ndim == 2\n",
    "        assert s >= 0\n",
    "        assert s <= 1\n",
    "        nnz, size, sparse = countnnZ(A, s, bytesPerVar=bytesPerVar)\n",
    "        nnzList.append(nnz)\n",
    "        sizeList.append(size)\n",
    "        hasSparse = (hasSparse or sparse)\n",
    "\n",
    "    totalnnZ = np.sum(nnzList)\n",
    "    totalSize = np.sum(sizeList)\n",
    "    if expected:\n",
    "        return totalnnZ, totalSize, hasSparse\n",
    "    numNonZero = 0\n",
    "    totalSize = 0\n",
    "    hasSparse = False\n",
    "    for i in range(len(matrixList)):\n",
    "        A, s = matrixList[i], sparcityList[i]\n",
    "        numNonZero_ = np.count_nonzero(A)\n",
    "        numNonZero += numNonZero_\n",
    "        hasSparse = (hasSparse or (s < 0.5))\n",
    "        if s <= 0.5:\n",
    "            totalSize += numNonZero_ * 2 * bytesPerVar\n",
    "        else:\n",
    "            totalSize += A.size * bytesPerVar\n",
    "    return numNonZero, totalSize, hasSparse\n",
    "\n",
    "# W, B, Z are tensorflow graph nodes\n",
    "W, B, Z, _ = protoNN.getModelMatrices()\n",
    "matrixList = ([W, B, Z])\n",
    "sparcityList = [SPAR_W, SPAR_B, SPAR_Z]\n",
    "nnz, size, sparse = getModelSize(matrixList, sparcityList)\n",
    "# print(\"Final test accuracy\", acc)\n",
    "print(\"Model size constraint (Bytes): \", size)\n",
    "# print(\"Number of non-zeros: \", nnz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0bd4e0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # summarize the fit of the model\n",
    "\n",
    "# print(\"Classification summary: \\n\",metrics.classification_report(Y_test, y_pred))\n",
    "# print(\"Confusion matrix: \\n\",metrics.confusion_matrix(Y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}