src/05b-ng-sim-eeap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Similarity - w/ AttentionMM (doc similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras import backend as K\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout, Lambda\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalMaxPooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import custom_attn\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "DOCSIM_IDLABELS = os.path.join(DATA_DIR, \"docsim-idlabels.tsv\")\n",
    "DOCSIM_TEXTS = os.path.join(DATA_DIR, \"docsim-texts.tsv\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 2\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 5\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1885 (40, 60)\n"
     ]
    }
   ],
   "source": [
    "docid2mat = {}\n",
    "ftext = open(DOCSIM_TEXTS, \"rb\")\n",
    "for line in ftext:\n",
    "    rec_id, text = line.strip().split(\"\\t\")\n",
    "    M = np.zeros((MAX_SENTS, MAX_WORDS))\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = pad_or_truncate(nltk.word_tokenize(sent), MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            M[sid, wid] = word_id\n",
    "    docid2mat[int(rec_id)] = M\n",
    "ftext.close()\n",
    "print(len(docid2mat), docid2mat[list(docid2mat.keys())[0]].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Label and DocID pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70253, 2) (70253, 2)\n"
     ]
    }
   ],
   "source": [
    "xdata, ydata = [], []\n",
    "fidl = open(DOCSIM_IDLABELS, \"rb\")\n",
    "for line in fidl:\n",
    "    label, docid_left, docid_right = line.strip().split(\"\\t\")\n",
    "    xdata.append((int(docid_left), int(docid_right)))\n",
    "    ydata.append(int(label))\n",
    "X = np.array(xdata)\n",
    "Y = to_categorical(np.array(ydata), num_classes=NUM_CLASSES)\n",
    "print(X.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partition into training, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(44259, 2) (44259, 2) (4918, 2) (4918, 2) (21076, 2) (21076, 2)\n"
     ]
    }
   ],
   "source": [
    "Xtv, Xtest, Ytv, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "Xtrain, Xval, Ytrain, Yval = train_test_split(Xtv, Ytv, train_size=0.9)\n",
    "print(Xtrain.shape, Ytrain.shape, Xval.shape, Yval.shape, \n",
    "      Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 40, 60) (64, 40, 60) (64, 2)\n"
     ]
    }
   ],
   "source": [
    "def datagen(X, Y, docid2mat, batch_size=BATCH_SIZE):\n",
    "    while True:\n",
    "        num_recs = X.shape[0]\n",
    "        indices = np.random.permutation(np.arange(num_recs))\n",
    "        num_batches = num_recs // batch_size\n",
    "        for bid in range(num_batches):\n",
    "            batch_ids = indices[bid * batch_size : (bid + 1) * batch_size]\n",
    "            Xbatch_l = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            Xbatch_r = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            for idx, (docid_l, docid_r) in enumerate(X[batch_ids, :]):\n",
    "                Xbatch_l[idx] = docid2mat[docid_l]\n",
    "                Xbatch_r[idx] = docid2mat[docid_r]\n",
    "            Ybatch = Y[batch_ids, :]\n",
    "            yield [Xbatch_l, Xbatch_r], Ybatch\n",
    "\n",
    "train_gen = datagen(Xtrain, Ytrain, docid2mat)\n",
    "[Xbatch_left, Xbatch_right], Ybatch = train_gen.next()\n",
    "print(Xbatch_left.shape, Xbatch_right.shape, Ybatch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 60, 300)           12219000  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200)               240600    \n",
      "=================================================================\n",
      "Total params: 12,459,600\n",
      "Trainable params: 12,459,600\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_in = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                       output_dim=WORD_EMBED_SIZE,\n",
    "                       weights=[E])(sent_in)\n",
    "\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                               return_sequences=False))(sent_emb)\n",
    "\n",
    "sent_model = Model(inputs=sent_in, outputs=sent_enc)\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sum_over_axis(X, axis):\n",
    "    return K.mean(X, axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_3 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 40, 200)       12459600    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistribu (None, 40, 200)       12459600    input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional)  (None, 40, 100)       75300       time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional)  (None, 40, 100)       75300       time_distributed_2[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "attention_mm_1 (AttentionMM)     (None, 200)           76000       bidirectional_2[0][0]            \n",
      "                                                                   bidirectional_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 200)           800         attention_mm_1[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 200)           0           batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 50)            10050       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 50)            0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 2)             102         dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 12,697,152\n",
      "Trainable params: 12,696,752\n",
      "Non-trainable params: 400\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LHS document\n",
    "doc_in_l = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_l = TimeDistributed(sent_model)(doc_in_l)\n",
    "\n",
    "doc_enc_l = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_l)\n",
    "\n",
    "# RHS document\n",
    "doc_in_r = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_r = TimeDistributed(sent_model)(doc_in_r)\n",
    "\n",
    "doc_enc_r = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_r)\n",
    "\n",
    "# Attention\n",
    "doc_att = custom_attn.AttentionMM(\"concat\")([doc_enc_l, doc_enc_r])\n",
    "doc_att = BatchNormalization()(doc_att)\n",
    "\n",
    "# Prediction\n",
    "fc1_dropout = Dropout(0.2)(doc_att)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=[doc_in_l, doc_in_r], outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "691/691 [==============================] - 944s - loss: 0.1604 - acc: 0.9233 - val_loss: 0.0829 - val_acc: 0.9788\n",
      "Epoch 2/5\n",
      "691/691 [==============================] - 939s - loss: 0.0412 - acc: 0.9891 - val_loss: 0.0565 - val_acc: 0.9846\n",
      "Epoch 3/5\n",
      "691/691 [==============================] - 939s - loss: 0.0363 - acc: 0.9909 - val_loss: 0.1127 - val_acc: 0.9648\n",
      "Epoch 4/5\n",
      "691/691 [==============================] - 939s - loss: 0.0328 - acc: 0.9915 - val_loss: 0.0424 - val_acc: 0.9885\n",
      "Epoch 5/5\n",
      "691/691 [==============================] - 939s - loss: 0.0292 - acc: 0.9927 - val_loss: 0.0504 - val_acc: 0.9860\n"
     ]
    }
   ],
   "source": [
    "train_gen = datagen(Xtrain, Ytrain, docid2mat, batch_size=BATCH_SIZE)\n",
    "val_gen = datagen(Xval, Yval, docid2mat, batch_size=BATCH_SIZE)\n",
    "\n",
    "num_train_steps = len(Xtrain) // BATCH_SIZE\n",
    "num_val_steps = len(Xval) // BATCH_SIZE\n",
    "\n",
    "history = model.fit_generator(train_gen, \n",
    "                              steps_per_epoch=num_train_steps,\n",
    "                              epochs=NUM_EPOCHS,\n",
    "                              validation_data=val_gen,\n",
    "                              validation_steps=num_val_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VGX2+PHPIQlEpIgBhSVgYG2AKAgiigq2NaACihS7\n/lTWtSBrRd111bWwXbF8XQt2BAUVVBAbWLEERQVB2qIERELoSiDl/P54bsgwTJJJMjP3zuS8X695\nZWbuM5kzF3LPPM8993lEVTHGGGOCpoHfARhjjDGRWIIyxhgTSJagjDHGBJIlKGOMMYFkCcoYY0wg\nWYIyxhgTSJagjDHGBJIlKGOMMYFkCcqYABLH/j5NvWZ/AMZUQUTGiMgyEdkiIt+JyBkh2y4TkYUh\n2w73nm8nIi+LSIGIFIrIg97zt4vIcyGvzxERFZF07/FsEblbRD4GfgU6isjFIe+xXER+HxbfIBGZ\nJyKbvThzRWSoiMwNa3etiEyN354yJvbS/Q7AmIBbBhwLrAGGAs+JyP7AMcDtwGAgD/gtUCwiacDr\nwHvA+UAp0LMG73c+0B/4HhDgIOA0YDlwHDBDRL5Q1S9FpBfwDHAW8C7QBmgK/A/4r4h0UtWFIb/3\nrtrsAGP8Yj0oY6qgqi+p6mpVLVPVScASoBdwKfB3Vf1CnaWq+oO37TfADar6i6oWqepHNXjLp1R1\ngaqWqGqxqr6hqsu893gfeAuXMAEuAcar6ttefKtUdZGqbgcmAecBiEgXIAeXOI1JGpagjKmCiFzg\nDaFtFJGNwCFAS6AdrncVrh3wg6qW1PItV4a9f38R+VRE1nvvP8B7//L3ihQDwNPAOSIiuN7Ti17i\nMiZpWIIyphIish/wGHAVkKWqewHzcUNvK3HDeuFWAu3LzyuF+QVoHPK4dYQ2O5cXEJFGwBTgn8C+\n3vtP996//L0ixYCqfgrswPW2zgGejfwpjQkuS1DGVG5PXMIoABCRi3E9KIDHgetFpIdXcbe/l9A+\nB34CxorIniKSKSJ9vNfMA44TkfYi0hy4uZr3bwg08t6/RET6A78L2f4EcLGInCgiDUSkrYgcHLL9\nGeBBoLiGw4zGBIIlKGMqoarfAf8C5gA/A12Bj71tLwF3AxOALcCrwN6qWgqcDuwP/AjkA8O917yN\nOzf0DTCXas4JqeoWYBTwIrAB1xOaFrL9c+Bi4D/AJuB9YL+QX/EsLqE+hzFJSGzBQmNSk4jsAawF\nDlfVJX7HY0xNWQ/KmNT1B+ALS04mWdl1UMakIBFZgSumGOxzKMbUmg3xGWOMCSQb4jPGGBNIgRvi\na9mypebk5PgdhjHGmDiZO3fuOlVtVV27wCWonJwc8vLy/A7DGGNMnIjID9G0syE+Y4wxgRS4HpQx\nxpiAUYWSEiguBhHYY4+EvK0lKGOMqQ1VKC11B+2SkooDePj96rZHc9/P31FcDGVlFZ/7jDPg5ZcT\nsostQRlTX6hWHFTLyqL7WZO2sf4Zi98RzwN8aal//5YZGe6Wnl7xM5r7jRpBkyY1f13o/YMOStjH\ntARlTKiyMtixA7Zv3/1npOeq+xlNm/KDXbwP2KHfglOBCKSlQYMGlf8MPZBXdsBt2BAaN675gbq2\nB/i6/o4G9ad0wBKUSbzS0rod9OOZMEpqu4xTJdLS3AGwUaPdf5bfz8ioaFfVwTbSz5q0jdXPILxn\ngwYuQZmUZgnKROfnn+GBB2DjxronjFgPjZR/Cw496Ef62bQptGxZfcKoyc/q2qSlxfazGlOPWIIy\n1Vu5Ek46CZYtg+bNqz4oh2+Px0E//Gc9GvIwVSsuhnfegUmTYPFiV2xW2a1x49ptb9jQOm+JYgnK\nVG3ZMjjxRNiwAd5/H/r0qf41xiRQaan7rzlpEkyZAoWF7ntSjx6u075xI2zbtuvt119rP5rboEF8\nEl9l2+pzQrQEZSr33Xeu57RjB7z3nvuLNyYAyspgzhyYOBEmT4Y1a2DPPWHQIBg+HE45xXWwq1JS\nsnviKk9ekZ6PZvuvv8KmTZG3FRfX7rOWX3YU68RX2bYgJURLUCayL790f+Xp6e7raZcufkdk6jlV\nmDvXJaVJkyA/HzIz4dRTXVI69VR3sI1Wero7Ldm0afxiDlVZQowm6VWVMOOVECtLYH37wl/+Ett9\nUxlLUGZ3n3wCAwa4cZJ334X99/c7IlNPqcK337qENHEiLF/uih5POQXGjoWBAxOXYOoqSAmxLknx\n118TEz9Ygqr3tm+HggJ3W7sWCmYvoODfU1m7598oOOpcCq5r4p4vcP8x+/SB/v0hNxd+8xu/ozep\n6vvvK3pKCxe6YsgTToBbb3UTGbRo4XeEwZfohBgPlqBSTHnCKU8q1d3fsiX8N3QB/kZ6mbLPh0Kr\nVtCqFXTs6E4Ov/eeG/MH6NbNJav+/eGoo9wfhDG19b//uYQ0aRLMm+eGmo47Dq6+GoYMgX328TtC\nk2iBW1G3Z8+easttVKh7wnEyMtiZbFq1cn/su9xf/BGt/nMr+3TKotXUx2neYe+IJ0pV4ZtvYMYM\nd/v4Y1dF1bw5nHyy9a5MzaxaBS++6HpLn3/unuvdG0aMgKFD7f9RqhKRuaras9p20SQoEckF7gfS\ngMdVdWzY9v2A8UArYD1wnqrme9v+DpyKW9rjbeAareJNUz1BFRVVJJTwBFPbhLNbsolwv3nzKipz\nnnwSLr0Ujj4a3ngDmjWL+vNs2uSuOylPWKtXu+cPO8wlqwEDrHdldrV2reuFT5wIH33kvvR07+6S\n0rBhYOuVpr6YJSgRSQMWAycD+cAXwNmq+l1Im5eA11X1aRE5AbhYVc8XkaOBfwDHeU0/Am5W1dmV\nvV+yJaiqEk6k+wlJODXx4INuDOXkk+GVV1ytbi2Vn9CePt16V2ZX69e7CbAnTXLDxGVl0LmzS0rD\nh8OBB/odoUmkaBNUNN9rewFLVXW594snAoOA70LadAau9e7PAl717iuQCTQEBMgAfo7mA/gl1gmn\nPKl07Fh14mnWzIdrD8aOhZtvdhePTJpU/YUj1RCBQw91tzFjdu9dlZ+7Ku9dlZ+7ysiIwWcxgbN5\nM0yd6npKb73lqsr23x9uucUlpUMO8TtCE3TRJKi2wMqQx/nAkWFtvgbOxA0DngE0FZEsVZ0jIrOA\nn3AJ6kFVXVj3sKMXmnCiOY+T1AknWqrw5z/D3XfDOefAU0/FJUs0b+5Obg8ZUtG7Kk9W//yny4/N\nm7trgcsTlvWuktsvv7hR4okTXU96+3Zo3x7++EeXlA4/PMB/FyZwYnVm4HrgQRG5CPgAWAWUisj+\nQCcg22v3togcq6ofhr5YREYCIwHat29fp0BWrHDDBtEknNCksv/+VRcRBDrh1ISqO1rcf7877/TI\nIwmZ0DS0d3XTTbv3rqZMce2sd5V8tm+HN990SWnaNHc5QuvW8Pvfu6TUu7dNl2hqJ5pzUEcBt6vq\nKd7jmwFU9d5K2jcBFqlqtojcAGSq6l+9bbcBRar698rer67noAoK4Nxzqz+PkzIJpyZKS91R44kn\nYPRo+Pe/A7ETwntXH3/shoOaNdv13FXbtn5HasqFTsr6yituOC8rC846y31BPPZYm8jdVC6WRRLp\nuCKJE3E9oy+Ac1R1QUiblsB6VS0TkbuBUlW9TUSGA5cBubghvjeB+1T1tcreL9mKJJJGcTFccIH7\nmvvnP8MddwQiOUWyaZObwGL6dPfNfNUq9/yhh7qqQOtd+aN8UtaJE12Pd/16N0R75pmup3TCCfZv\nYqIT6zLzAcB9uDLz8ap6t4jcCeSp6jQROQu4F1cU8QFwpapu9yoAH8ZV8SnwpqpeG/ldHEtQcVBU\n5I4g06bB3/4GN97od0RRs96Vv0InZX3pJbcsWE0nZTUmXEwTVCJZgoqxX36BwYPdeMxDD8EVV/gd\nUZ2U967KE1Zo76r83NXRR9s3+bpQhby8ilkd6jopqzHhLEEZdzQ/9VT3FXj8eLjwQr8jiilVmD9/\n1+uurHdVO1VNyjpiRHJNymqCzxJUfbdunTs6f/01TJjg5o1JcZs371oZaL2r6lU2KeuIETYpq4kf\nS1D12U8/uS7E0qXubPapp/odUcKV967Kk9VHH1X0rk46yRVb1NfeVfmkrBMnuu8v5ZOyDh9uk7Ka\nxLAEVV/98IM7Av/0E7z2Ghx/vN8RBUJ9713l57siB5uU1QSBJaj6aMkSOPFEdzSeMcPVYpvdVNe7\nKk9Yyd67+vlnN73UpEnwoXdpvE3KaoLAElR9M3++O7qWlrqJz7p39zuipLF5866Vgfn57vmuXStm\nZE+W3lX5pKwTJ8KsWTYpqwkmS1D1SV6eK7fKzHTjWJ06+R1R0krG3lVlk7KWJyWblNUEjSWo+uKj\nj9xX/Kws1w3o2NHviFJKUHtXv/wCr7/uhu9CJ2UdPtwmZTXBZwmqPnj7bXdJf/v2rueUnV39a0yt\nqcKCBS5RTZ8euXeVmxu/f4aiIjf106RJu07KOmyYTcpqkoslqFQ3dao7Mh18sEtUVhuccNX1rvr3\nhz596ta7Kp+UdeJEePVVm5TVpAZLUKnshRfg/POhRw93ZNx7b78jqvdCe1czZriquZISN/tC6Lmr\naHpXNimrSXWWoFLV44/DyJHuq/Prr9v8MwG1ZUvFjOzR9K7KyuCTT9zwnU3KalKdJahUdN99brHB\n3Fz31dpm7EwK4b2rjz5yQ3flvavsbLemUuikrCNGuAIM+yc2qcgSVCpRhXvugT/9yU2Q9sIL9nU6\niZX3rsqLLX7+2SZlNfVLtAkqVku+m3hRhZtvdus4nXcePPkkpNs/WzJr2tStgDJ4sPvnLS6Ghg39\njsqY4LEjXZCVlcGoUW4dp9//Hh5+2OqIU4yIJSdjKhPV0U5EckXkexFZKiJjImzfT0TeFZFvRGS2\niGSHbGsvIm+JyEIR+U5EcmIXfgorKYFLLnHJ6brr4P/+z5KTMaZeqfaI5y3b/hDQH+gMnC0incOa\n/RN4RlUPBe7ELf9e7hngH6raCegFrI1F4Cltxw445xx46im4/Xb4xz9sWgBjTL0TzVfyXsBSVV2u\nqjuAicCgsDadgfe8+7PKt3uJLF1V3wZQ1a2q+mtMIk9V27a5C15eegn++U/4y18sORlj6qVoElRb\nYGXI43zvuVBfA2d6988AmopIFnAgsFFEXhaRr0TkH16PbBciMlJE8kQkr6CgoOafIlVs3epqjKdP\nd0N6113nd0TGGOObWJ3UuB7oKyJfAX2BVUAprgjjWG/7EUBH4KLwF6vqo6raU1V7tmrVKkYhJZmN\nG+F3v3NTCDz9NFx+ud8RGWOMr6Kp4lsFtAt5nO09t5OqrsbrQYlIE2CIqm4UkXxgnqou97a9CvQG\nnohB7KmjoMAlpwUL3NDemWdW/xpjjElx0fSgvgAOEJEOItIQGAFMC20gIi1FpPx33QyMD3ntXiJS\n3i06Afiu7mGnkFWroG9fWLTITVFtyckYY4AoEpSqlgBXATOBhcCLqrpARO4UkYFes37A9yKyGNgX\nuNt7bSlueO9dEfkWEOCxmH+KZLViBRx3HKxc6dZRyM31OyJjjAkMm+rIL99/Dyee6Faee/NNOPJI\nvyMyxpiEsKmOguybb+Dkk908N7Nnw2GH+R2RMcYEjk1NkGiffw79+rl1Fj780JKTMcZUwhJUIr3/\nvhvWa9HCJaeDDvI7ImOMCSxLUIlSXgSRnQ0ffAAdOvgdkTHGBJolqER4+WW30M/BB7vk1DZ8Ig5j\njDHhLEHF23PPwbBh0KMHzJoF9XWmDGOMqSGr4oun//4X/vAHVxQxbRo0aeJ3RMaYACguLiY/P5+i\noiK/Q4mrzMxMsrOzycjIqNXrLUHFy7/+Bddf7yZ/fekl2GMPvyMyxgREfn4+TZs2JScnB0nR1QpU\nlcLCQvLz8+lQy3PuNsQXa6pwxx0uOQ0d6s4/WXIyxoQoKioiKysrZZMTgIiQlZVVp16i9aBiSRVu\nuMH1ni68EB5/HNJtFxtjdpfKyalcXT+j9aBipawMrrjCJacrr4Tx4y05GWNMHViCioWSErjoInjk\nEbjpJnjgAWhgu9YYE0wbN27k4YcfrvHrBgwYwMaNG+MQUWR2FK2rHTtgxAh49lm46y64915bot0Y\nE2iVJaiSkpIqXzd9+nT22muveIW1GxuDqott22DIEJgxA/7zHxg92u+IjDGmWmPGjGHZsmV069aN\njIwMMjMzadGiBYsWLWLx4sUMHjyYlStXUlRUxDXXXMPIkSMByMnJIS8vj61bt9K/f3+OOeYYPvnk\nE9q2bcvUqVPZI8YFYZagamvLFjj9dDczxGOPwaWX+h2RMSYZjR4N8+bF9nd26wb33Vfp5rFjxzJ/\n/nzmzZvH7NmzOfXUU5k/f/7OcvDx48ez9957s23bNo444giGDBlCVlbWLr9jyZIlvPDCCzz22GMM\nGzaMKVOmcN5558X0Y1iCqo0NG6B/f8jLg+efh7PP9jsiY4yptV69eu1yrdK4ceN45ZVXAFi5ciVL\nlizZLUF16NCBbt26AdCjRw9WrFgR87iiSlAikgvcD6QBj6vq2LDt++GWeW8FrAfOU9X8kO3NcEu9\nv6qqV8Uodn+sXQu/+x0sXAhTpsCgQX5HZIxJZlX0dBJlzz333Hl/9uzZvPPOO8yZM4fGjRvTr1+/\niNcyNWrUaOf9tLQ0tm3bFvO4qi2SEJE04CGgP9AZOFtEOoc1+yfwjKoeCtwJ3Bu2/a/AB3UP12f5\n+W6J9sWL4fXXLTkZY5JS06ZN2bJlS8RtmzZtokWLFjRu3JhFixbx6aefJji6CtH0oHoBS1V1OYCI\nTAQG4XpE5ToD13r3ZwGvlm8QkR7AvsCbQLVL/AbW8uVuLafCQpg5E4491u+IjDGmVrKysujTpw+H\nHHIIe+yxB/vuu+/Obbm5uTzyyCN06tSJgw46iN69e/sWZzQJqi2wMuRxPnBkWJuvgTNxw4BnAE1F\nJAvYAPwLOA84qbI3EJGRwEiA9u3bRxt74ixcCCedBEVF8N570DN586wxxgBMmDAh4vONGjVixowZ\nEbeVn2dq2bIl8+fP3/n89ddfH/P4IHbXQV0P9BWRr4C+wCqgFLgCmB56PioSVX1UVXuqas9WQVuO\nYt48N6xXWupWxLXkZIwxCRFND2oV0C7kcbb33E6quhrXg0JEmgBDVHWjiBwFHCsiVwBNgIYislVV\nx8Qk+nj79FNXrde0Kbz7LhxwgN8RGWNMvRFNgvoCOEBEOuAS0wjgnNAGItISWK+qZcDNuIo+VPXc\nkDYXAT2TJjnNmuWuc2rTBt55B/bbz++IjDGmXql2iE9VS4CrgJnAQuBFVV0gIneKyECvWT/gexFZ\njCuIuDtO8SbG9OkwYADk5LgLcS05GWNMwkV1HZSqTgemhz13W8j9ycDkan7HU8BTNY4w0SZPhnPO\nga5dXbVey5Z+R2SMMfWSTRYb6umnYfhwOOIIV61nyckYY3xjCarcww+7JTOOPx7eeguaN/c7ImOM\nCYQmTZr48r6WoAD+/ne3yODpp7sZIkKm/TDGGOOP+j1ZrCr85S/w17+6ob1nn4WMDL+jMsaYuBoz\nZgzt2rXjyiuvBOD2228nPT2dWbNmsWHDBoqLi7nrrrsY5PN0bvU3QanCdde5dZz+3/+DRx+FtDS/\nozLG1DM+rLbB8OHDGT169M4E9eKLLzJz5kxGjRpFs2bNWLduHb1792bgwIGIjwuw1s8EVVoKf/iD\nW8dp1CiXpGyJdmNMPdG9e3fWrl3L6tWrKSgooEWLFrRu3Zo//vGPfPDBBzRo0IBVq1bx888/07p1\na9/irH8JqrjYFUNMmAC33OKWabcl2o0xPvFrtY2hQ4cyefJk1qxZw/Dhw3n++ecpKChg7ty5ZGRk\nkJOTE3GZjUSqXwlq+3Z3rmnqVLjnHrj5Zr8jMsYYXwwfPpzLLruMdevW8f777/Piiy+yzz77kJGR\nwaxZs/jhhx/8DrEeJahff4UzznAl5OPGwdVX+x2RMcb4pkuXLmzZsoW2bdvSpk0bzj33XE4//XS6\ndu1Kz549Ofjgg/0OsZ4kqM2b4bTT4OOP4YknXFGEMcbUc99+++3O+y1btmTOnDkR223dujVRIe0i\n9RNUYSHk5roymQkT3BCfMcaYwEvtBLVmDZx8MixZAi+/7C7ENcYYkxRSN0H9+KNbBXfVKnjjDbdc\nuzHGBISq+nqNUSKoap1en5oX/yxdCsceCz//7IoiLDkZYwIkMzOTwsLCOh/Ag0xVKSwsJDMzs9a/\nI/V6UIsXQ9++7nqnWbPg8MP9jsgYY3aRnZ1Nfn4+BQUFfocSV5mZmWRnZ9f69amXoFq3hl693HVO\nXbr4HY0xxuwmIyODDh06+B1G4EU1xCciuSLyvYgsFZHdlmwXkf1E5F0R+UZEZotItvd8NxGZIyIL\nvG3xL6Fr1sxdiGvJyRhjklq1CUpE0oCHgP5AZ+BsEekc1uyfwDOqeihwJ3Cv9/yvwAWq2gXIBe4T\nkb1iFbwxxpjUFU0PqhewVFWXq+oOYCIQPgd7Z+A97/6s8u2qulhVl3j3VwNrgVaxCNwYY0xqi+Yc\nVFtgZcjjfODIsDZfA2cC9wNnAE1FJEtVC8sbiEgvoCGwLPwNRGQkMNJ7uFVEvo/6E1SuJbAuBr8n\nEZIl1mSJEyzWeLFY46O+xbpfNI1iVSRxPfCgiFwEfACsAkrLN4pIG+BZ4EJVLQt/sao+Cjwao1jK\n3zNPVXvG8nfGS7LEmixxgsUaLxZrfFiskUWToFYB7UIeZ3vP7eQN350JICJNgCGqutF73Ax4A7hV\nVT+NRdDGGGNSXzTnoL4ADhCRDiLSEBgBTAttICItRaT8d90MjPeebwi8giugmBy7sI0xxqS6ahOU\nqpYAVwEzgYXAi6q6QETuFJGBXrN+wPcishjYF7jbe34YcBxwkYjM827dYv0hKhHTIcM4S5ZYkyVO\nsFjjxWKND4s1AknlqTaMMcYkr9Sci88YY0zSswRljDEmkJI6QUUxBVMjEZnkbf9MRHISH+XOWKqL\n9SIRKQg5V3epH3F6sYwXkbUiMr+S7SIi47zP8o2I+DIjbxRx9hORTSH79LZExxgSSzsRmSUi33lT\nf10ToU1Q9ms0sQZi34pIpoh8LiJfe7HeEaFNII4DUcYapONAmoh8JSKvR9iWmH2qqkl5A9JwF/12\nxF0A/DXQOazNFcAj3v0RwKQAx3oR8KDf+9WL5TjgcGB+JdsHADMAAXoDnwU0zn7A637vTy+WNsDh\n3v2mwOII/weCsl+jiTUQ+9bbV028+xnAZ0DvsDZBOQ5EE2uQjgPXAhMi/Tsnap8mcw8qmimYBgFP\ne/cnAyeK+LJCWDSxBoaqfgCsr6LJINylA6ru2ra9vIuxEyqKOANDVX9S1S+9+1twFbFtw5oFZb9G\nE2sgePtqq/cww7uFV34F4jgQZayBIG7C71OBxytpkpB9mswJKtIUTOF/RDvbqCuX3wRkJSS6SuLw\nRIoVYIg3tDNZRNpF2B4U0X6eIDjKG1KZISKBmOLeGw7pjvsGHSpw+7WKWCEg+9YbipqHm+vzbVWt\ndL/6fByIJlYIxnHgPuBGYLeZfzwJ2afJnKBSzWtAjroZ4d+m4tuJqb0vgf1U9TDgAeBVn+Mpn2ll\nCjBaVTf7HU9Vqok1MPtWVUtVtRtulpteInKIX7FUJ4pYfT8OiMhpwFpVnZvo9w6XzAmq2imYQtuI\nSDrQHCgk8aKZLqpQVbd7Dx8HeiQottqIZt/7TlU3lw+pqOp0IENEWvoVj4hk4A74z6vqyxGaBGa/\nVhdr0PatF8dG3GoKuWGbgnIc2KmyWANyHOgDDBSRFbjTESeIyHNhbRKyT5M5QVU7BZP3+ELv/lnA\ne+qd1UuwaKaLCj3XMBA37h9U04ALvKqz3sAmVf3J76DCiUjr8nFxcbPpN8CnA5MXxxPAQlX9dyXN\nArFfo4k1KPtWRFqJt8aciOwBnAwsCmsWiONANLEG4Tigqjeraraq5uCOVe+p6nlhzRKyT5N2yXdV\nLRGR8imY0oDx6k3BBOSp6jTcH9mzIrIUdzJ9RIBjHSVu6qgSL9aL/IgVQERewFVptRSRfOAvuBO6\nqOojwHRcxdlS3KKUFwc0zrOAP4hICbANGOHTFxRw30rPB771zkEA3AK0h2DtV6KLNSj7tg3wtLiF\nVRvgpmJ7PYjHgShjDcxxIJwf+9SmOjLGGBNIyTzEZ4wxJoVZgjLGGBNIlqCMMcYEkiUoY4wxgWQJ\nyhhjTCBZgjLGGBNIlqCMMcYEkiUoY4wxgWQJyhhjTCBZgjLGGBNIlqCMMcYEkiUoY4wxgWQJypg4\nEZEVInKS33EYk6wsQRljjAkkS1DGGGMCyRKUMXEmIo1E5D4RWe3d7hORRt62liLyuohsFJH1IvKh\niDTwtt0kIqtEZIuIfC8iJ/r7SYxJrKRdUdeYJHIr0BvoBigwFfgT8GfgOiAfaOW17Q2oiBwEXAUc\noaqrRSQHtxqzMfWG9aCMib9zgTtVda2qFgB34JZUByjGLQW+n6oWq+qH3tLppUAjoLOIZKjqClVd\n5kv0xvjEEpQx8fcb4IeQxz94zwH8A1gKvCUiy0VkDICqLgVGA7cDa0Vkooj8BmPqEUtQxsTfamC/\nkMftvedQ1S2qep2qdgQGAteWn2tS1Qmqeoz3WgX+ltiwjfGXJShj4u8F4E8i0kpEWgK3Ac8BiMhp\nIrK/iAiwCTe0VyYiB4nICV4xRRGwDSjzKX5jfGEJypj4uwvIA74BvgW+9J4DOAB4B9gKzAEeVtVZ\nuPNPY4F1wBpgH+DmxIZtjL/EnY81xhhjgsV6UMYYYwLJEpQxxphAsgRljDEmkCxBGWOMCaTATXXU\nsmVLzcnJ8TsMY4wxcTJ37tx1qtqqunaBS1A5OTnk5eX5HYYxxpg4EZEfqm9lQ3zGGGMCKvUSVGkp\n3HknrF/vdyTGGGPqIPUS1DffwN13w7HHQn6+39EYY4yppcCdg6qz7t3hzTdh0CDo0wdmzoSDD/Y7\nKmOM2am4uJj8/HyKior8DiWuMjMzyc7OJiMjo1avT70EBXD88fD++5CbC8ccA9OnQ69efkdljDEA\n5Ofn07RgGQ2QAAAbwUlEQVRpU3JycnDzBKceVaWwsJD8/Hw6dOhQq9+RekN85bp3h48/hubN4YQT\n4K23/I7IGGMAKCoqIisrK2WTE4CIkJWVVadeYlQJSkRyReR7EVlavqBa2PbjRORLESkRkbPCtpWK\nyDzvNq3WkdbG/vvDRx+5n6edBhMnJvTtjTGmMqmcnMrV9TNWm6BEJA14COgPdAbOFpHOYc1+BC4C\nJkT4FdtUtZt3G1inaGujTRs33HfUUXDOOfDAAwkPwRhjTM1F04PqBSxV1eWqugOYCAwKbaCqK1T1\nG4K6oFrz5q5YYtAgGDUKbrsNbJkRY0w9tXHjRh5++OEav27AgAFs3LgxDhFFFk2CagusDHmc7z0X\nrUwRyRORT0VkcKQGIjLSa5NXUFBQg19dkygy4aWX4JJL4K9/hcsvd9dMGWNMPVNZgiopKanyddOn\nT2evvfaKV1i7SUQV336qukpEOgLvici3qrostIGqPgo8CtCzZ8/4dW3S0+Gxx2CffeDee2HdOnj+\neZe8jDGmnhgzZgzLli2jW7duZGRkkJmZSYsWLVi0aBGLFy9m8ODBrFy5kqKiIq655hpGjhwJVExF\nt3XrVvr3788xxxzDJ598Qtu2bZk6dSp77LFHTOOMJkGtAtqFPM72nouKqq7yfi4XkdlAd2BZlS+K\nJxG45x6XpP74RxgwAF59FZo18y0kY0w9Nno0zJsX29/ZrRvcd1+lm8eOHcv8+fOZN28es2fP5tRT\nT2X+/Pk7y8HHjx/P3nvvzbZt2zjiiCMYMmQIWVlZu/yOJUuW8MILL/DYY48xbNgwpkyZwnnnnRfT\njxHNEN8XwAEi0kFEGgIjgKiq8USkhYg08u63BPoA39U22JgaPRqeew4+/BD69YOff/Y7ImOM8UWv\nXr12uVZp3LhxHHbYYfTu3ZuVK1eyZMmS3V7ToUMHunXrBkCPHj1YsWJFzOOqtgelqiUichUwE0gD\nxqvqAhG5E8hT1WkicgTwCtACOF1E7lDVLkAn4L8iUoZLhmNVNRgJCuDccyErC4YMcbNOvPUWdOzo\nd1TGmPqkip5Oouy5554778+ePZt33nmHOXPm0LhxY/r16xfxWqZGjRrtvJ+Wlsa2bdtiHldU56BU\ndTowPey520Luf4Eb+gt/3SdA1zrGGF+5ufDuu3DqqS5JvfkmHHaY31EZY0zcNG3alC1btkTctmnT\nJlq0aEHjxo1ZtGgRn376aYKjq5C6M0nURO/e7oLe9HQ47jj44AO/IzLGmLjJysqiT58+HHLIIdxw\nww27bMvNzaWkpIROnToxZswYevfu7VOUIBqw64F69uypvi1YuHIl/O538L//waRJ7ropY4yJsYUL\nF9KpUye/w0iISJ9VROaqas/qXms9qFDt2rmeVLducOaZMH683xEZY0y9ZQkqXFYWvPMOnHyyu6h3\n7FibdcIYY3xgCSqSJk1g2jQ4+2y4+Wa47jooC+YsTsYYk6pScz2oWGjY0F0n1aoV/Oc/UFDghvxq\nufCWMcaYmrEEVZUGDdw1CvvuC7feCoWFbj6/kGsGjDHGxIcN8VVHBG65xc3hN3MmnHQSrF/vd1TG\nGJPyLEFF69JLYfJk+OorOPZYyM/3OyJjjEmIJk2a+PK+lqBq4owzXC8qPx+OPhoWLfI7ImOMSVmW\noGqqb1+3Qu+OHXDMMfDZZ35HZIwxNTJmzBgeeuihnY9vv/127rrrLk488UQOP/xwunbtytSpU32M\n0LGZJGpr2TI368SaNfDyy3DKKX5HZIxJEqGzK/iw2gZfffUVo0eP5v333wegc+fOzJw5k+bNm9Os\nWTPWrVtH7969WbJkCSJCkyZN2Lp1a61isZkk/PDb38LHH8OBB8Jpp8ELL/gdkTHGRKV79+6sXbuW\n1atX8/XXX9OiRQtat27NLbfcwqGHHspJJ53EqlWr+NnnZYiszLwuWreG2bPdnH3nnOOulRo1yu+o\njDFJxK/VNoYOHcrkyZNZs2YNw4cP5/nnn6egoIC5c+eSkZFBTk5OxGU2Esl6UHXVvLlbomPwYLjm\nGvjTn2xqJGNM4A0fPpyJEycyefJkhg4dyqZNm9hnn33IyMhg1qxZ/PDDD36HaAkqJjIz3QW8l14K\nd98Nv/89lJb6HZUxxlSqS5cubNmyhbZt29KmTRvOPfdc8vLy6Nq1K8888wwHH3yw3yHaEF/MpKfD\no4+6WSfuvhvWrYMJE1zyMsaYAPr222933m/ZsiVz5syJ2K62BRJ1ZT2oWBKBu+6C+++HV15xq/Vu\n2uR3VCbAyspg+3a/ozAmmCxBxcOoUfD8867Kr18/8LkSxgTPli3w4IPQqRM0awZ//KPrdBtjKliC\nipdzzoHXXoPFi6FPH1i+3O+ITAAsXeque2nbFq6+Glq0gGHDYNw46NjRjQ7/8ovfUZpECNo1qPFQ\n189oCSqecnPhvfdgwwY3NVKsr8YzSUEV3n7bXS534IHw8MMwcCB8+qm7PfsszJ8PJ57oikD33x8e\neQSKi/2O3MRLZmYmhYWFKZ2kVJXCwkIy63Ae3maSSISFC91ME5s2uYUQ+/b1OyKTAFu3uuTzwAPu\nv8A++8Dll7tbmzaRX/PJJ3DTTfDRR3DAAXDPPTBkiDu9aVJHcXEx+fn5vl9nFG+ZmZlkZ2eTEbaO\nXrQzSViCSpSVK12SWr4cJk50102ZlPS//8FDD8Hjj7vvJD16uEvkhg2DRo2qf70qvPEGjBkDCxZA\nr17wt7+505nGpAKb6iho2rWDDz90k2QNGeKOXiZlqLrR3MGD3SxY998P/fu7HtEXX8D550eXnMD1\nlk47Db7+Gp58ElavhuOPhwED3HPG1BdRJSgRyRWR70VkqYiMibD9OBH5UkRKROSssG0XisgS73Zh\nrAJPSllZ8O67bpLZyy6De++1WSeS3K+/usvfunZ155A+/titb7lihZue8aijaj88l5YGF13k6mz+\n8Q93vqp7d5fsVqyI4YcwJqhUtcobkAYsAzoCDYGvgc5hbXKAQ4FngLNCnt8bWO79bOHdb1HV+/Xo\n0UNT3o4dqueeqwqq11yjWlrqd0SmhlasUL3xRtUWLdw/Y7duqk8+qbptW/zec8MG1TFjVDMzVRs2\nVB09WrWgIH7vZ0y8AHlaTe5R1ah6UL2Apaq6XFV3ABOBQWFJboWqfgOUhb32FOBtVV2vqhuAt4Hc\nmqXQFJSRAc884+qN77/ffSXescPvqEw1VN1SYEOGuJLwf/3L9Zo+/BC+/NL1duI5cchee7lO95Il\ncMEFVppuUl80CaotsDLkcb73XDSieq2IjBSRPBHJKygoiPJXJ7kGDeDf/3ZHnAkTXN2xHWUCads2\nGD/eDa/16+cmsL/xRlfv8tJLbt3KRFbZZWfDY4/tXpr+3/9aabpJLYEoklDVR1W1p6r2bNWqld/h\nJI6IK9V67DF3ocyJJ0Jhod9RGc/Kle58Urt2cMklblqixx+H/Hz3vaJ9e3/j69TJzaj18ccuQV1+\nOXTpApMn26lNkxqiSVCrgHYhj7O956JRl9fWH5deClOmuAt5jz3WHRmNL1TdNUjDhkGHDq68+7jj\nYNYsV0F3ySWwxx5+R7mro4+GDz5wl9g1bAhDh0Lv3q6nZ0wyiyZBfQEcICIdRKQhMAKYFuXvnwn8\nTkRaiEgL4Hfecybc4MEwcyasWuWOOAsX+h1RvVJUBE8/DT17uu8Ib78N114Ly5bByy+7ob0gXywr\nAqefbqXpJrVUm6BUtQS4CpdYFgIvquoCEblTRAYCiMgRIpIPDAX+KyILvNeuB/6KS3JfAHd6z5lI\n+vZ1X4WLi92Jjc8+8zuilLd6Nfz5z2647qKLXKJ65BE3jPf3v0NOjt8R1oyVpptUYjNJBNHy5e5a\nqZ9+ckN/uVb4GEuqLvePG+eKHEpLXe9j1Cg44YRg95RqasMGN0x5//3uHNoVV8Ctt0LLln5HZuoz\nm0kimXXs6M58H3igO3JOmOB3RClh+3Z47jk48kh3Ae0bb7gZxZcuhalTXY1KKiUncLOljx27a2n6\nb39rpekmOViCCqp993VnuY85Bs49130FNrWyZg3ccQfst58b6tq82c2Vt2qVq/Tv2NHvCOMvtDT9\nhBOsNN0kB0tQQda8OcyYAWee6S7qvfVWqx+ugfI58Nq3h9tvdwUQM2fCd9+5oa4mTfyOMPEilaYf\ncoiVpptgsgQVdJmZ8OKLMHKkW3th5EgoKfE7qsAqLnaTxR99tJsFfOpU+MMfXNHA66+7U3sN7H/9\nLqXp6elWmm6CKeX+VLdvhxtugEmT3LmFlPhWmJbmSsv+9Cd3pejQoa7czOy0di3cdZerujv7bLd8\n+rhxrhrv/vvd2kpmV+Wl6d9842bKsNJ0EzQpV8X33Xdw+OEuUYGbv6xHD3fr2dPdcnKS+GT4Aw+4\ncrO+fV33oHlzvyPy1ZdfukT0wgtuOsNTTnG7JzfXeko1tW2bOzd3zz2wcSOcdx7ceWfyldqb4KvX\nCxYWF7uF3vLy3G3uXPeNsPxk8N57VySs8p/t2ydR0nrhBbjwQujcGd58E1q39juihCouhldfdYnp\no49gzz3dtT9XXQUHH+x3dMnPStNNvNXrBBXJ9u2ugmnu3IrE9e23FadzWrbctZfVo4erfAps0nrr\nLVc8se++7v5vf+t3RHG3bp2rRHv4YTd017GjKxO/+OJ635GMi/x8V1zy5JOuoOTGG12tzp57+h2Z\nSXaWoKJQVOSSVHkvKy/PJbHSUrd9n312TVg9e8JvfpOQ0KLz+efuhEF6uqv2697d74ji4uuv3cjm\n88+7f7OTTnLDeAMGuNNzJr4WLnST5r76quus3347/L//51aNMaY2LEHV0rZt7qRx6PDgggVuqAOg\nTZvde1q+jrAtWuRK0zZtcuek+vXzMZjYKSlxFWbjxrk1mBo3dheaXn21G9k0iffJJ64XVX4N+d13\nu7WxAjvKYALLElQM/fqrm2g8dHhw4cKKCsG2bXftZfXo4XpfCZOf75LU8uVu1okzz0zgm8fW+vWu\nUPGhh+DHH93FtVdd5WYRb9HC7+iMqivXHzPGFST16uXOV6XI9yKTIJag4mzrVpe0Qnta339fkbTa\ntdu1l9WjR5xPMhcWwmmnuWG/Rx6Byy6L45vF3vz5bhjv2WddL/b4490w3umn2zBeEJWWukWhb7vN\nfT/q39+tkXXYYX5HZpKBJSgfbN4MX321a09ryZKK7Tk5u/e0Ytor+OUXd43UjBnuoqBbbgn0+Etp\nqfs2Pm4cvPeeuyb5/PPdMF7Xrn5HZ6JhpemmNixBBcSmTe5andBCjGXLKrZ37LhrT+vww921W7VW\nXOzOYD/3nOuC/Oc/gbsgaONGd2Hogw/C//7neptXXunWbczK8js6UxvhpelXXum+H1lpuonEElSA\nbdhQkbTKb6Fr9RxwwK6FGN27Q7NmNXiDsjK4/nqXnM4+G556yi216rOFC90w3tNPu/N6xx4L11wD\ngwa5QkST/Kw03UTDElSSKSys6GGV//zxx4rtBx206/Bg9+7VTHaq6lbcGzPGFVBMmeLL7KhlZW7E\n8f773Sq1jRrBOee4YbwUrYo3uAKKW25xhaVWmm7CWYJKAQUFu57PmjvXfUMFd2rp4IN3HR7s1i3C\nN9Xx413BRM+ebgGkBI25bNrkOm4PPOCGNNu2dTMSXHYZtGqVkBBMAISXpt9zjysyDfCpUZMAlqBS\n1Jo1LlGFJq6ffnLbGjRw1wiFDg8edhjs8dZUGDHC1Wy/9Zab1ylOFi92Sempp1yl49FHu2G8M86w\nb8/1lZWmm3CWoOqR1at3TVh5eW52b3Al2l26QM/sNfR87+/0aLaEQ9/8O5ndO8Xs/cvKXN4bN84N\n5zVs6PLh1Ve7JGkMRC5NHzsWDj3U78hMolmCqsdU3Wqxoeez8vLcXHYA6RTT9cDt9Diuyc6e1iGH\nuPNDNbFliyt4eOAB13Nq3dqtvfT737spAo2JxErTjSUoswtVWLkS8t5YQ96trzB30wHk7Xkc67e4\n6r6MDPdNNrQQ45BDIg/LLV3qDjDjx7trv4480lW0n3VWIIoFTZKw0vT6yxKUqdzPP0P//ug337Li\nHy8xt93gXQoxNm50zRo1qkhaPXu6ZUqeeMLVWqSnw7BhbhjvyCP9/TgmuVlpev1jCcpUbfNmGDwY\nZs1y10uNHg24ntby5btWDs6d65qDm2Pw8svdrU0bH+M3KcdK0+sPS1CmekVF7gTAlCmuxOqeeyLW\n/5aVuWG9H390F9fW9FyVMTXx8cdw001Wmp7Kok1QUc2BIyK5IvK9iCwVkTERtjcSkUne9s9EJMd7\nPkdEtonIPO/2SE0/iImjzEyYNMlVNYwd6+YaKl/BMUSDBu5AcdJJlpxM/PXpAx9+6HpS6enu3Gbv\n3jB7tt+RmUSrNkGJSBrwENAf6AycLSLhK/JcAmxQ1f2B/wB/C9m2TFW7ebfLYxS3iZW0NPi//3O1\nv+PHu6PBtm1+R2XqOREYONCtzTZ+vLuU4vjj3SKV33zjd3T1T1GRuwZz0SJ3CiBRopkBrRewVFWX\nA4jIRGAQ8F1Im0HA7d79ycCDItYhTxoicMcdboqHUaPglFPcaoF1mrXWmLpLS4OLL3bX1T34oBvu\n69bNStNrQtUtdLBxo7tt2lRxP9rH27dX/L4zzoCXX05M7NEkqLbAypDH+UB43dbONqpaIiKbgPJ5\nqTuIyFfAZuBPqvph+BuIyEhgJED7OM5yYKpx1VWuxveCC6BvX3jzTauEMIGwxx5www1uFLq8NH3S\npPpRml5W5oqUokkmlbUpLa36PRo1ct9HQ285ORX3mzevuN+xY0I+NhBFkYSInAXkquql3uPzgSNV\n9aqQNvO9Nvne42W4JLYFaKKqhSLSA3gV6KKqmyt7PyuSCIC333Zfk/bZx00Rsf/+fkdkzC7CS9Nv\nuslNqRXE0vTi4oqkUZvey+bNFQuhVmbPPXdNLqEJpbrHzZu709GJFG2RRDQ9qFVAu5DH2d5zkdrk\ni0g60BwoVJf9tgOo6lwvcR0IWAYKspNPduXn/fu7M9YzZriFqowJiOxsePxxuPZa14O69VY3o0k8\nStOLimqfXDZudMNrVRFxy+lU1nupLsE0a5a6pfjR9KDSgcXAibhE9AVwjqouCGlzJdBVVS8XkRHA\nmao6TERaAetVtVREOgIfeu3WV/Z+1oMKkEWL3PmoDRtcSdXxx/sdkTERVVaaDhXnX2qbYELPv0SS\nlrb78FhNejBNmwZuTdG4i+l1UCIyALgPSAPGq+rdInInkKeq00QkE3gW6A6sB0ao6nIRGQLcCRQD\nZcBfVPW1qt7LElTA5OdDbq5bu/6ZZ9z9xo1T9yubSVqq8NprcPPN7qLfZs1ccqrp+ZeaJJe99nJ/\nDlYSVjN2oa6JnfXr4bTTYM6ciufS092Z68aNd/8Z6bmatAl/Li3Nv89ukk5pKTz3HHz+eTDPvxhL\nUCbWfv3VlU1t2ODub9sW+WdV26obK6lMw4bxT4KNG7sjVX0bazHGB7EskjDGHcAvvrhuv6O01J1x\nrmliq6rNunWRtxUX1y7GzMzaJ7iatGnUyMaFjKmGJSiTOGlprh42EbXAJSV1T37h2376KfK2srKa\nxyeya8+tUaPobg0bxqdterolTBM4lqBMakpPd+VRTZvG931UXW+trslv+3Z327Gj4n75JfyV3Xbs\niN3nEIk+ocUrSYa3zciwpFnPWYIypi7KD+wNGyZ+aqjy5FhVEouU+OrSLjRpVtY2lhKVJKP9fZY0\nE8oSlDHJKjQ5xrunGK3Kkma0CbI2bbdsceciq2oXS7FKeLHqsTZsmLJJ0xKUMSZ2gpo0S0oS29Pc\nutVdnlFVm1hWUGdkJK432aYNdO0au9irYAnKGJPaRNwBPCPDTdwXBKFJM5qkGIvE+csvFUmzsnbR\nFPwMHgyvvBL/fYQlKGOMSbzQpBkkoT3NypJYAs+1WoIyxhjjpKe7W0CmhbfL5o0xxgSSJShjjDGB\nFLi5+ESkAPghBr+qJbAuBr8nEZIl1mSJEyzWeLFY46O+xbqfqraqrlHgElSsiEheNJMRBkGyxJos\ncYLFGi8Wa3xYrJHZEJ8xxphAsgRljDEmkFI5QT3qdwA1kCyxJkucYLHGi8UaHxZrBCl7DsoYY0xy\nS+UelDHGmCRmCcoYY0wgJXWCEpFcEfleRJaKyJgI2xuJyCRv+2cikpP4KHfGUl2sF4lIgYjM826X\n+hGnF8t4EVkrIvMr2S4iMs77LN+IyOGJjtGLo7o4+4nIppB9eluiYwyJpZ2IzBKR70RkgYhcE6FN\nUPZrNLEGYt+KSKaIfC4iX3ux3hGhTSCOA1HGGqTjQJqIfCUir0fYlph9qqpJeQPSgGVAR6Ah8DXQ\nOazNFcAj3v0RwKQAx3oR8KDf+9WL5TjgcGB+JdsHADMAAXoDnwU0zn7A637vTy+WNsDh3v2mwOII\n/weCsl+jiTUQ+9bbV028+xnAZ0DvsDZBOQ5EE2uQjgPXAhMi/Tsnap8mcw+qF7BUVZer6g5gIjAo\nrM0g4Gnv/mTgRBFfVvaKJtbAUNUPgPVVNBkEPKPOp8BeItImMdFViCLOwFDVn1T1S+/+FmAh0Das\nWVD2azSxBoK3r7Z6DzO8W3jlVyCOA1HGGggikg2cCjxeSZOE7NNkTlBtgZUhj/PZ/Y9oZxtVLQE2\nAVkJia6SODyRYgUY4g3tTBaRdokJrVai/TxBcJQ3pDJDRLr4HQyANxzSHfcNOlTg9msVsUJA9q03\nFDUPWAu8raqV7lefjwPRxArBOA7cB9wIVLZAVEL2aTInqFTzGpCjqocCb1Px7cTU3pe4Ob8OAx4A\nXvU5HkSkCTAFGK2qm/2OpyrVxBqYfauqparaDcgGeonIIX7FUp0oYvX9OCAipwFrVXVuot87XDIn\nqFVA6LeLbO+5iG1EJB1oDhQmJLpK4vDsFquqFqrqdu/h40CPBMVWG9Hse9+p6ubyIRVVnQ5kiEhL\nv+IRkQzcAf95VX05QpPA7NfqYg3avvXi2AjMAnLDNgXlOLBTZbEG5DjQBxgoIitwpyNOEJHnwtok\nZJ8mc4L6AjhARDqISEPcibppYW2mARd6988C3lPvrF6CVRtr2LmGgbhx/6CaBlzgVZ31Bjap6k9+\nBxVORFqXj4uLSC/c/3dfDkxeHE8AC1X135U0C8R+jSbWoOxbEWklInt59/cATgYWhTULxHEgmliD\ncBxQ1ZtVNVtVc3DHqvdU9bywZgnZp0m7oq6qlojIVcBMXJXceFVdICJ3AnmqOg33R/asiCzFnUwf\nEeBYR4nIQKDEi/UiP2IFEJEXcFVaLUUkH/gL7oQuqvoIMB1XcbYU+BW4OKBxngX8QURKgG3ACJ++\noID7Vno+8K13DgLgFqA9BGu/El2sQdm3bYCnRSQNlyRfVNXXg3gciDLWwBwHwvmxT22qI2OMMYGU\nzEN8xhhjUpglKGOMMYFkCcoYY0wgWYIyxhgTSJagjDHGBJIlKGOMMYFkCcoYY0wg/X/4N7PZ+zdZ\n2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2ea2398090>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.498\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[5424 5230]\n",
      " [5348 5054]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "test_gen = datagen(Xtest, Ytest, docid2mat, batch_size=BATCH_SIZE)\n",
    "num_test_steps = len(Xtest) // BATCH_SIZE\n",
    "\n",
    "Ytest_ = model.predict_generator(test_gen, num_test_steps)\n",
    "\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)[0:len(ytest_)]\n",
    "\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}