diff --git a/README.md b/README.md index c0e4778..d5dcd41 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ DNC: Differentiable Neural Computer Implements DeepMind's third nature paper, [Hybrid computing using a neural network with dynamic external memory](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html) by Graves et. al. -![DNC schema](copy/static/dnc_schema.png?raw=true) +![Repeat copy results](static/repeat_copy_results.png?raw=true) Based on the paper's appendix, I sketched the [computational graph](https://docs.google.com/drawings/d/1Fc9eOH1wPw0PbBHWkEH39jik7h7HT9BWAE8ZhSr4hJc/edit?usp=sharing) -Based on Mostafa-Samir's code, I got the copy task to work ([Jupyter notebook](https://nbviewer.jupyter.org/github/greydanus/dnc/blob/master/copy/copy.ipynb)) +I got the repeat-copy copy task to work ([Jupyter notebook](https://nbviewer.jupyter.org/github/greydanus/dnc/blob/master/repeat-copy/repeat-copy-nn.ipynb)) _This is a work in progress_ -------- diff --git a/dnc/controller.py b/dnc/controller.py index fe245a8..3be816d 100755 --- a/dnc/controller.py +++ b/dnc/controller.py @@ -63,7 +63,7 @@ class Controller(): ''' raise NotImplementedError("nn_step does not exist") - def zero_state(self): + def get_state(self): ''' Returns the initial state of the controller. If the controller is not recurrent, it still needs to return a dummy value Returns: LSTMStateTensor or another type of state tensor @@ -78,7 +78,7 @@ class Controller(): controller_dim: the output dimension of the controller ''' test_chi = tf.zeros([self.batch_size, self.chi_dim]) - nn_output, nn_state = self.nn_step(test_chi, state=None) + nn_output, nn_state = self.nn_step(test_chi, state=self.get_state()) return nn_output.get_shape().as_list()[-1] def prepare_interface(self, zeta_hat): diff --git a/dnc/controller.pyc b/dnc/controller.pyc index ccaafc7..4752c27 100644 Binary files a/dnc/controller.pyc and b/dnc/controller.pyc differ diff --git a/dnc/dnc.py b/dnc/dnc.py index 67f5195..f0ea1f3 100755 --- a/dnc/dnc.py +++ b/dnc/dnc.py @@ -44,7 +44,7 @@ class DNC: self.X_tensor_array = self.unstack_time_dim(self.X) # initialize states - nn_state = self.controller.zero_state() + nn_state = self.controller.get_state() dnc_state = self.memory.zero_state() # values for which we want a history @@ -63,7 +63,8 @@ class DNC: (_, next_nn_state, next_dnc_state, dnc_hist) = output # write down the history - with tf.control_dependencies(next_dnc_state): + controller_dependencies = [self.controller.update_state(next_nn_state)] + with tf.control_dependencies(controller_dependencies): self.dnc_hist = {self.hist_keys[i]: self.stack_time_dim(v) for i, v in enumerate(dnc_hist)} # convert to dict def step(self, time, nn_state, dnc_state, dnc_hist): diff --git a/dnc/dnc.pyc b/dnc/dnc.pyc index 03d1dfa..cbb0599 100644 Binary files a/dnc/dnc.pyc and b/dnc/dnc.pyc differ diff --git a/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb deleted file mode 100644 index 41f68e5..0000000 --- a/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb +++ /dev/null @@ -1,459 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Repeat Copy Task\n", - "### Using a Differentiable Memory Architecture (DNC)\n", - "\n", - "\"DNC\n", - "\n", - "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", - "\n", - "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '../dnc')\n", - "\n", - "from dnc import DNC\n", - "from nn_controller import NNController\n", - "\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Hyperparameters" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "xydim = 6\n", - "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", - "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", - "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", - "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", - "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", - "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", - "tf.app.flags.DEFINE_integer(\"iterations\", 40000, \"Number of training iterations\")\n", - "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", - "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", - "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", - "tf.app.flags.DEFINE_string(\"save_path\", \"models/model.ckpt\", \"Where to save checkpoints\")\n", - "FLAGS = tf.app.flags.FLAGS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Data functions" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAD4lJREFUeJzt3X+MHOV9x/H39y7mnJjYVu3UtkoaSEiK7g7R2A4pTQw0\ntmVEVNNKlcMVqQISIUPSP65qS5CRnFIX0kbEtFBH/WU7FeHcVE1URw0Y29CmiFAXO6G5MxSFH4GW\ns4tBOJLxufbd0z9mj97t7ezud3ZmZ2f285L2j5195tnvM8+z8+zuzHzHQgiIiIhIcfXkHYCIiIi0\nRpO5iIhIwWkyFxERKThN5iIiIgWnyVxERKTgNJmLiIgUnCZzERGRgtNkLiIiUnCazEVERAruXVlW\nbmZLgA3Ay8BElu8lIiJSMvOBC4F9IYQ36hVMNJmb2eeB3wWWA88Avx1C+PcaRTcA30jyHiIiIgLA\nDcBD9Qq4J3Mz+wxwL3ALcAgYBvaZ2UdCCCeqir8McN2Dv8rSS5bMemH/8EHWb187p/6e1btd8Uw9\nfaOrfK+zfq+4TPf7iL7ZVJs6fJOvfmcq/Z2rfV1889PnnPXPi3nlEeAaV11z+Rr72afPusqnNdby\nGsvu+lMaa/uHD7B++7o5y7Mfa9nVH1937U+ud6x59zveO2Z4+zauc1Mby5nv1+L2O7HvELM8pn8P\n+8ZmdvGfAL4Flbm0niS/zIeBvwgh/C2AmW0GPg3cDPxJVdkJgKWXLGHFyuWzXuhb3DdnGfgP4k/V\nqKOeXmf9XnF9Oh9YUWO5N37/fXF8g37FSt9OCs6LWR7XYg9fY1es/F9X+bTGWl5jOev648ba/Jj2\nZj/Wsqw/ru7a49g71rz7Hfdk7uzbuM7Nayz792tx+53Yd4hZHte/vrGZffyND1O7+sjM5gGrgIPT\ny0J027UDwBXe6ERERKR13i9cS4m+ZB6vWn6c6Pi5iIiItJkuTRMRESk47zHzE8AksKxq+TLgWNxK\n+4cP0re4b9ayRT+/0PnWxTaYdwBt1z0tHri+P+8Q2qq72ts94xi6rW+hs/r3R8Bo1bLmr+h2TeYh\nhLNmdhhYC+wFMDOrPP+zuPXWb18bc8JM9+ikIdMel+YdQNsMDnXXDnBgaCDvENqouz653TaWO6t/\nL2XufnMc+Mum1k5yNvtXgd2VSX360rT3ALsT1CUiIiItck/mIYRvmtlS4C6iv9d/CGwIIbyednAi\nIiLSWKIMcCGEHcCOlGMRERGRBHQ2u4iISMElSee6Bvg9ouQxK4BfCyHsbbBS9GjC1OQXXfH09HzZ\nVd7LnYlpyhe/V5Ob8R1bJn2ZjP6ox5sm0av5LbplypshzMe5Kd1jzV1/r6/+rMean6/F/rHmrL+3\n+fqzHmvu/Yh3P2j3+N7AOTg7b6z5tqh3P7jNnGPTu1+eai6D4PiRs+xc3VydSX6ZLyA6Tn4b/jEq\nIiIiKUtyAtwjRHfRmL4sTURERHKkY+YiIiIFp8lcRESk4DSZi4iIFFyi68y99g8foG/R7NzsA0P9\nDHZVWkgREZHaxkbGGNtzdNayibfONL1+Wybz9dvXdX1udhERkTgDQwNz7nswfuQYO1fvamr9JNeZ\nLwAu5v+vrPugmV0GvBlCeNVbn4iIiLQmyS/z1cDjRNeYB+DeyvKvAzenFJeIiIg0Kcl15v+CTpwT\nERHpGJqURURECs71y9zM7gB+HbgEOA08CdweQni+7oohRI8mePNTe016cx5748k4KV6Tm/EdnvzU\nSWyZbC7H8DRPPP5N6VthsuPyTTtlPNbc9wFoMt90J3JvSufnMOs8/e7wvYm4Oy7ZZ3Z5+pPU792g\nzW5Oz2b3/jJfA9wPfBxYB8wDHjWzdzvrERERkZS4fpmHEK6d+dzMbgT+h+gOak+kF5aIiIg0q9Vj\n5ouJ/nB6M4VYREREJIHEk3nljmn3AU+EEI42Ki8iIiLZaCUD3A6gH/hESrGIiIhIAokmczN7ALgW\nWBNCGG9Ufv/wQfoWV+Vmv76fwaH+JG8vIiJSKqMjY4yNzP6T+8zJDHOzVyby64CrQgivNLPO+u1r\nlZtdREQkxuDQwJybj40fOcbfrMogN7uZ7QCGgI3AKTNbVnnpZAhhwlOXiIiIpMN7AtxmYCHwz8Br\nMx6b0g1LREREmuW9zlzpX0VERDqMJmcREZGC8x4z3wzcClxYWTQG3BVCeCStgDJOkQw9vneYKnr+\n7oy3qDm/Dm6ZOutbwcOZH7mnx5t331d8KuP7AExN3eEq7+W+D0CPM/+1N/f7pG/sePJx3+kdl85c\n5e77ADi3fW/G97TIWub3AXDf0+I83wodwPvL/FXgdmAlUQrXx4C9ZqZrzERERHLiPWb+T1WL7jSz\nW4luvKIscCIiIjlInAHOzHqIzmLvA/41tYhERETEJUnSmEHg+8B84G1gUwjhx2kHJiIiIs1Jcjb7\nc8BlwOXAA8AeM/toqlGJiIhI09y/zEMI54AXK09/YGaXE53hfkvcOsrNLiIiEq/tudlr6AF66xVQ\nbnYREZF47c7NfjfwMPAK8F7gBuBKYJunHhEREUmP95f5zwJfB1YAJ4H/ADaEEB5POzARERFpjvc6\n889lFYiIiIgko9zsIiIiBdfSCXBm9kXgbuC+EMLv1CnYdPLdyYzzWbt5kwZnzJ3DeNKXw9iTzzqJ\nDtucLt5c617mzB/d03OPq3zWudy9vLnW3ULzg22bM6/8nc7PVdZ59737TW8u96zHmvs+AM79lDf3\nvn+/6cvlvs2ajb/5KTrxL3Mz+xjR5WjPJK1DREREWpdoMjez84EHgc8Bb6UakYiIiLgk/WX+58B3\nQgiPpRmMiIiI+CXJzX498IvA6vTDERERES9v0pgLgPuAdSGEjM9eERERkWZ4f5mvAt4HHDF757zk\nXuBKM/sC0BfC3PMS9w8foG9RVW72of45qetERES602jlMdNE02t7J/MDwKVVy3YDzwJfrjWRA6zf\nvk652UVERGINVh4zjQN/1dTa3gxwp4BZt3Uxs1PAGyGEZz11iYiISDrSyADnvNxfRERE0tTyLVBD\nCJ9KIxARERFJxmIOc6dTudlK4PBnD9+U3TFzb/xZ5xN1xpN1mkevTtucmeq0xnrj8aZ/fVe2Y63o\nm9PDHXvRN47GWl1ZjbXxI8fYuXoXwKoQwpF6ZV1/s5vZVjObqnocbbymiIiIZCXJ3+yjwFpg+rvO\nufTCEREREa8kk/m5EMLrqUciIiIiiSQ5m/3DZvbfZvaCmT1oZu9PPSoRERFpmncyfwq4EdgAbAYu\nAr5nZgtSjktERESa5E0as2/G01EzOwT8BNgE7Ipbr1Y610UfWMg192/wvH2hjY4cZXCoP+8w2mZ0\nZKxr0vV2U1uhu9o7NjLGQJe0FbSfytPYyBhje2afTz7x1pmm12/pOvMQwkkzex64uF65Wulc/27j\n37fy1oUztqe7PiRjI0c75kOStW5qK3RXe8f2HO2qyVz7qfwMDA3MGWszLk1rqKUMcGZ2PtFEPt5K\nPSIiIpKc9zrzr5jZlWb2ATP7ZeDbwFlgJJPoREREpCHv3+wXAA8BS4DXgSeAXwohvJF2YCIiItIc\n7wlwQ8765wOcePbEnBfOnDzD+JFjzupqBuUrn1NevzNv1W5vjzP8qTS2WR1pbc7U+jdLKTU2t7Hs\nTbGZ0liLa29BPoouEzGf27Kmc43bT+U11uJ02n4qq3SuJ557Z+6c36hs1rnZfxP4RmZvICIiUn43\nhBAeqlcg68l8CdE16S8DE5m9kYiISPnMBy4E9jU6nJ3pZC4iIiLZa+nSNBEREcmfJnMREZGC02Qu\nIiJScJrMRURECi6XydzMPm9mL5nZaTN7ysw+lkccWTOzrWY2VfU42njNzmdma8xsb+V2uFNmtrFG\nmbvM7DUze9vM9ptZ3Rz+naxRe81sV42+/m5e8bbCzO4ws0Nm9lMzO25m3zazj9QoV4r+baa9Zelf\nM9tsZs+Y2cnK40kzu6aqTCn6FRq3tyz9CjlM5mb2GeBeYCvwUeAZYJ+ZLW13LG0yCiwDllcen8w3\nnNQsAH4I3EaNlBJmdjvwBeAW4HLgFFE/n9fOIFNUt70VDzO7r71JljrFGuB+4OPAOmAe8KiZvXu6\nQMn6t2F7K8rQv68CtwMrgVXAY8BeM+uH0vUrNGhvRRn6FUIIbX0Q3RP9T2c8N+C/gN9vdyxtaOtW\n4EjecbShnVPAxqplrwHDM54vBE4Dm/KON6P27gK+lXdsGbV3aaXNn+yS/q3V3jL37xvATWXv15j2\nlqZf2/rL3MzmEX07Oji9LERb9ABwRTtjaaMPV/6afcHMHjSz9+cdUNbM7CKib7gz+/mnwL9R3n4G\nuLryN+1zZrbDzH4m74BSspjo34g3oSv6d1Z7ZyhV/5pZj5ldD/QB3yt7v1a3d8ZLpejXlu5nnsBS\noBc4XrX8OPALbY6lHZ4CbgT+E1gBfInoQzMYQjiVY1xZW060M6zVz8vnFi+Fh4F/AF4CPgTcA3zX\nzK6ofGEtJDMz4D7giRDC9Pkepe3fmPZCifrXzAaB7xNlF3ub6Ff3C2Z2BSXs17j2Vl4uTb+2ezLv\nKiGEfTOejprZIeAnwCaiv3ekJEII35zxdMzMfgS8AFwNPJ5LUOnYAfQDn8g7kDap2d6S9e9zwGXA\nIuA3gD1mdlW+IWWqZntDCD8oU7+2+wS4E8Ak0ckGMy0DOvwWW60LIZwEngcKe3Zok44RnQvRlf0M\nEEJ4iWi8F7avzewB4Frg6hDC+IyXStm/ddo7R5H7N4RwLoTwYmUy20L0N/qtlLRf67S3VtnC9mtb\nJ/MQwlngMLB2elnlb621wJPtjCUPZnY+0SCpu6MousoH4hiz+3kh0dnCpe9nADO7AFhCQfu6MrFd\nB/xKCOGVma+VsX/rtTemfKH7t0oP0FvGfo3RQ3S4d44i92sef7N/FdhtZoeBQ8Aw8B5gdw6xZMrM\nvgJ8h+iv9Z8D/gA4C4zkGVcazGwB0ReT6TsFf9DMLgPeDCG8SnTc8U4z+zHRXfP+kOiqhX/MIdyW\n1Wtv5bGV6NjbsUq5Pyb6F2bf3No6m5ntILo8ZyNwysymf6mdDCFM3/2wNP3bqL2Vvi9F/5rZ3UTH\niV8B3gvcAFwJbKsUKU2/Qv32lqlfgfZfmlY5p+A2ooFymujEhNV5n9afUTtHiD4Ip4kG00PARXnH\nlVLbriK6fGey6rFzRpkvEV3q8jbRh+PivOPOor1EJ9Y8QrRDmABeBL4GvC/vuBO2tVY7J4HfqipX\niv5t1N4y9S/w15X4T1fa8yjwqTL2a6P2lqlfQwi6BaqIiEjRKTe7iIhIwWkyFxERKThN5iIiIgWn\nyVxERKTgNJmLiIgUnCZzERGRgtNkLiIiUnCazEVERApOk7mIiEjBaTIXEREpOE3mIiIiBafJXERE\npOD+D2xMPvufese9AAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def get_sequence(length, reps, dim):\n", - " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", - " X = np.vstack(X) ; X[:,dim-1] = 0\n", - " \n", - " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", - " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", - " markers = range(length+1, X.shape[0], 2*length+3)\n", - " X[markers,dim-1] = 1\n", - " return X, y\n", - " \n", - "def next_batch(batch_size, length, reps, dim):\n", - " X_batch = []\n", - " y_batch = []\n", - " for _ in range(batch_size):\n", - " X, y = get_sequence(length, reps, dim)\n", - " X_batch.append(X) ; y_batch.append(y)\n", - " return [X_batch, y_batch]\n", - "\n", - "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", - "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Helper functions" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def binary_cross_entropy(y_hat, y):\n", - " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", - "\n", - "def llprint(message):\n", - " sys.stdout.write(message)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Build graph, initialize everything" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "building graph...\n", - "defining loss...\n", - "computing gradients...\n", - "init variables... \n", - "ready to train..." - ] - } - ], - "source": [ - "sess = tf.InteractiveSession()\n", - "\n", - "llprint(\"building graph...\\n\")\n", - "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", - "dnc = DNC(NNController, FLAGS)\n", - "\n", - "llprint(\"defining loss...\\n\")\n", - "y_hat, outputs = dnc.get_outputs()\n", - "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", - "loss = binary_cross_entropy(y_hat, dnc.y)\n", - "\n", - "llprint(\"computing gradients...\\n\")\n", - "gradients = optimizer.compute_gradients(loss)\n", - "grad_op = optimizer.apply_gradients(gradients)\n", - "\n", - "llprint(\"init variables... \\n\")\n", - "sess.run(tf.global_variables_initializer())\n", - "llprint(\"ready to train...\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model overview...\n", - "\tvariable \"W1:0\" has 2048 parameters\n", - "\tvariable \"b1:0\" has 128 parameters\n", - "\tvariable \"W2:0\" has 32768 parameters\n", - "\tvariable \"b2:0\" has 256 parameters\n", - "\tvariable \"W_z:0\" has 12288 parameters\n", - "\tvariable \"W_v:0\" has 1536 parameters\n", - "\tvariable \"W_r:0\" has 60 parameters\n", - "total of 49084 parameters\n" - ] - } - ], - "source": [ - "# tf parameter overview\n", - "total_parameters = 0 ; print \"model overview...\"\n", - "for variable in tf.trainable_variables():\n", - " shape = variable.get_shape()\n", - " variable_parameters = 1\n", - " for dim in shape:\n", - " variable_parameters *= dim.value\n", - " print '\\tvariable \"{}\" has {} parameters' \\\n", - " .format(variable.name, variable_parameters)\n", - " total_parameters += variable_parameters\n", - "print \"total of {} parameters\".format(total_parameters)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "loaded model: models/model.ckpt-5000\n" - ] - } - ], - "source": [ - "global_step = 0\n", - "saver = tf.train.Saver(tf.global_variables())\n", - "load_was_success = True # yes, I'm being optimistic\n", - "try:\n", - " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", - " ckpt = tf.train.get_checkpoint_state(save_dir)\n", - " load_path = ckpt.model_checkpoint_path\n", - " saver.restore(sess, load_path)\n", - "except:\n", - " print \"no saved model to load.\"\n", - " load_was_success = False\n", - "else:\n", - " print \"loaded model: {}\".format(load_path)\n", - " saver = tf.train.Saver(tf.global_variables())\n", - " global_step = int(load_path.split('-')[-1]) + 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Train loop" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 5100/40000\n", - "\tloss: 0.0911\n", - "Iteration 5200/40000\n", - "\tloss: 0.0515\n", - "Iteration 5300/40000\n", - "\tloss: 0.0327\n", - "Iteration 5400/40000\n", - "\tloss: 0.0205\n", - "Iteration 5500/40000\n", - "\tloss: 0.0244\n", - "Iteration 5600/40000\n", - "\tloss: 0.1585\n", - "Iteration 5700/40000\n", - "\tloss: nan\n", - "Iteration 5800/40000\n", - "\tloss: nan\n", - "Iteration 5900/40000\n", - "\tloss: nan\n", - "Iteration 6000/40000\n", - "\tloss: nan\n", - "\n", - "\tSAVING MODEL\n", - "Iteration 6100/40000\n", - "\tloss: nan\n", - "Iteration 6200/40000\n", - "\tloss: nan\n", - "Iteration 6300/40000\n", - "\tloss: nan\n", - "Iteration 6400/40000\n", - "\tloss: nan\n", - "Iteration 6500/40000\n", - "\tloss: nan\n", - "Iteration 6600/40000\n", - "\tloss: nan\n", - "Iteration 6700/40000\n", - "\tloss: nan\n", - "Iteration 6800/40000\n", - "\tloss: nan\n", - "Iteration 6900/40000\n", - "\tloss: nan\n", - "Iteration 7000/40000\n", - "\tloss: nan\n", - "\n", - "\tSAVING MODEL\n", - "Iteration 7100/40000\n", - "\tloss: nan\n", - "Iteration 7200/40000\n", - "\tloss: nan\n", - "Iteration 7300/40000\n", - "\tloss: nan\n", - "Iteration 7391/40000" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtsteps\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtsteps\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mstep_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mloss_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "loss_history = []\n", - "for i in xrange(global_step, FLAGS.iterations + 1):\n", - " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", - "\n", - " rlen = np.random.randint(1, FLAGS.length + 1)\n", - " rreps = np.random.randint(1, FLAGS.reps + 1)\n", - " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", - " tsteps = rreps*(2*rlen+3)\n", - "\n", - " fetch = [loss, grad_op]\n", - " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", - "\n", - " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", - " loss_history.append(step_loss)\n", - "\n", - " if i % 100 == 0:\n", - " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", - " loss_history = []\n", - " if i % FLAGS.save_every == 0 and i is not 0:\n", - " llprint(\"\\n\\tSAVING MODEL\\n\")\n", - " saver.save(sess, FLAGS.save_path, global_step=i)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python2.7/site-packages/matplotlib/colors.py:938: UserWarning: Warning: converting a masked element to nan.\n", - " vmin = float(vmin)\n", - "/usr/local/lib/python2.7/site-packages/matplotlib/colors.py:939: UserWarning: Warning: converting a masked element to nan.\n", - " vmax = float(vmax)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqcAAAKECAYAAAA63ZOcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3X2YZVV94PvvrwmhFaG4AQMYfAccGAhapSBKNypMmmEy\nbbzxokdMxhBCQOU69dxcX0YHFL1OEh9tRkknJrkjYMvh6jWM7VVoo6CAiGgVokgTH0Fe5E1eYpE0\nkoD9u3/sXdSp6nrr6qqzdtX+fp5nP91nnb1P/dY6q8751dp7rR2ZiSRJktQEq0oHIEmSJI0zOZUk\nSVJjmJxKkiSpMUxOJUmS1Bgmp5IkSWoMk1NJkiQ1hsmpJEmSGsPkVJIkSY1hcipJkqTGMDmVJElS\nY5icSpIkqTFMTiVJktQYJqeSJElqDJNTSWqAiHhVRGyPiNdO89yb6ueOKRGbJPVTZGbpGCRJQETc\nCXw7M0+ZUv4l4JDMPLRMZJLUP46cSlJzbAJ+OyL2Gi+IiP2Afwd8ulhUktRHJqeS1BwXA6uB1/eU\nvRHYDfhMkYgkqc88rS9JDRIR3wb+KTNPrB9fB2RmvrJsZJLUH46cSlKzXAwcHxHPiogXAi/HU/qS\nWsSRU0lqkIjYF7gX+C/A04H3As/KzEeKBiZJfWJyKkkNExH/E3ge1fWnt2bm75SNSJL6x9P6ktQ8\nFwO/CRxCNYNfklrDkVNJapiI2B24HwjggMz818IhSVLf/ErpACRJO9gOPAl8wcRUUtt4Wl+Smud1\nwH5Up/clqVU8rS9JDRERRwNHAe8DfpaZLysckiT1nSOnktQcZwF/QXW96X8qHIskFeHIqSRJkhrD\nkVNJkiQ1hrP1Z1DfpWUdcAfweNloJEmS+mI11U1AtmTmwyUCMDmd2TrgM6WDkCRJKuBU4JISP7h1\nyWlEvA34E+AA4Cbg7Mz8zjS73lH9879SregynSuAkxY9xpVnqdpp4ddL/xF/s6Dj/oYzFvwzZ2df\nmh/bacJs/X8L1d/XOzpjgX1/V+3K7Ia/4Y8WdNxcv+czt9JS/q4vRyvp927hPXG2352lbKGFRrzw\nPvwQ8HfwVB7Uf61KTiPiDcBHgTOAG4BhYEtEHJqZD03ZvT6Vvx9w4AyvuHqW5zRhqdpp4R8yC49m\nqd5v+9L82E4TZuv/M7dTqdbbtam3C4t6rqNm7032swkr6fduab43lrKFSnzT1Ypd0ti2CVHDwCcz\n8+LMvBU4E3gMOK1sWJIkSYIWJaf1vaqHgK+Nl2W1jtZXgWNLxSVJkqQJrUlOqc7P7wY8MKX8Aarr\nTyVJklRYm5LTJXBE6QCWCdtpbrbR/NhO82M7zYetNF+21FxsocXVpglRDwG/BPafUr4/1a0CZ3AF\n1aXOvY4Ajqw3zc12mpttND+20/z4VTkfttJ8+Xs3l+XbQj8Abp5SVn5p99Ykp5n5RESMACcAmwEi\nIurHH5/5yJNYObMUJUmSxk030HYf8NcFYpnQmuS09jHgwjpJHV9K6unAhSWDkiRJUqVVyWlmfjYi\n9gPOozqd/z1gXWY+WDYySZIkQcuSU4DM3AhsLB2HJEmSduRsfUmSJDWGyakkSZIaw+RUkiRJjdG6\na04lgFjgcefwgQX/zPM4Z8HHnsN5Cz52oc7j3L7/TK1cC/2dgzL9X02XCzrKvrQ8tGrkNCLWRMTm\niLgnIrZHxPrSMUmSJGlCq5JTYE+q5aPeykL/7JIkSdKSadVp/cy8gup+pON3h5IkSVKDtG3kVJIk\nSQ1mcipJkqTGMDmVJElSY7TqmtOFuQJYPaXsCODIArFIkiQtlh8AN08pe7xEIJOYnM7pJODA0kFI\nkiQtsiPZcbDtPuCvC8QyoVXJaUTsCRzMxHrQL4iIo4BHMvPucpFJkiQJWpacAi8FrqJa4zSBj9bl\nFwGnlQpKkiRJlVYlp5n5DZwEJkmS1FgmapIkSWoMk1NJkiQ1RqtO60vjcoHHncc5C/6Z53Dego/d\nFbsSs7RYFvo7Bwvvw6V+59QPC7sD+XL8PDx3gf34HD6woOPuA/5mQUcuHkdOJUmS1BitSU4j4j0R\ncUNEPBoRD0TEZRFxaOm4JEmSNKE1ySmwBvgEcAxwIrA78JWIeFrRqCRJkvSU1lxzmpkn9z6OiLcA\nPwOGgGtLxCRJkqTJ2jRyOtU+VNfoP1I6EEmSJFVamZxGRADnA9dm5i2l45EkSVKlNaf1p9gIHA68\nsnQgkiRJmtC65DQiLgBOBtZk5n1zH3EFsHpK2RHAkYsemyRJUr/cXG+9Hi8RyBStSk7rxPS1wPGZ\nedf8jjoJOHAJo5IkSeq/I+qtVxMW4W9NchoRG4EOsB7YFhH710+NZWYT/lCQJElqvTZNiDoT2Bv4\nOnBvz3ZKwZgkSZLUozUjp5nZpkRckiRpWTJhkyRJUmOYnEqSJKkxIjNLx9BIETEIjMAZOFt/JVpY\nvz+H8xb8E2PBR+6ahf6Gn8e5ixqHmmL5febvyu/dQtn/taOF/+7sSh8+j3MWfOzCPDVffygzR/v8\nwwFHTiVJktQgrUlOI+LMiLgpIsbq7bqIOKl0XJIkSZrQmuQUuBt4FzAIDAFXApsj4vCiUUmSJOkp\nbVpK6ktTit4XEWcBxwC3FAhJkiRJU7QmOe0VEauoFt/fA7imcDiSJEmqtSo5jYgjgG8Bq4HHgFMy\n88dlo5IkSdK4ViWnwK3AUcAA8Hrg0og4PjNvnPmQK6hy2V5HAEcuUYiSJEn9cHO99Xq8RCCTtCo5\nzcwngdvrhzdGxNHAWVSLmc7gJFznVJIkrTxH1Fuvp9Y5LaZNs/WnswrYrXQQkiRJqrRm5DQiPgxc\nDtwF7AWcCqwFPlQyLkmSJE1oTXIK/DpwEdU5+jHg+8C6zLyqaFSSJEl6SmuS08w8vXQMkiRJml3b\nrzmVJElSg5icSpIkqTFac1pfK1Eu+MhzOG9Bx53HOQv+mecu8GfCrtR012KWei309wbgPM7dhWMX\n1od3JV413cI+FUv1iV35HF7499VCf+digcctntaOnEbEuyNie0R8rHQskiRJqrQyOY2Il1EtvH9T\n6VgkSZI0oXXJaUQ8A9gEnA78vHA4kiRJ6tG65BT4C+CLmXll6UAkSZI0WasmREXEG4EXAy8tHYsk\nSZJ21JrkNCIOAs4HTszMJ0rHI0mSpB21JjkFhoBnAqMRMb5Owm7A2oh4O7BHZk6zNsUVwOopZUcA\nRy5dpJIkSUvuB8DNU8oeLxHIJG1KTr/KjhnlhcBW4E+nT0wBTgIOXMq4JEmSCjiSHVOj+4C/LhDL\nhNYkp5m5DbiltywitgEPZ+bWMlFJkiSpVxtn6/falRvvSJIkaZG1ZuR0Opn5mtIxSJIkaULbR04l\nSZLUICankiRJagyTU0mSJDVGzLiCUstFxCAwAmfgUlIr0UL7fcy9y6L/zF21KzFLvXalD5foh8st\nXi295fg53O/vq6eWkhrKzNEFvsguac3IaUScGxHbp2y3zH2kJEmS+qVts/VvBk5g4s+JJwvGIkmS\npCnalpw+mZkPlg5CkiRJ02vNaf3aIRFxT0TcFhGbIuLZpQOSJEnShDYlp9cDbwHWAWcCzweujog9\nSwYlSZKkCa05rZ+ZW3oe3hwRNwB3AqcAnyoTlSRJknq1JjmdKjPHIuJHwMGz73kFsHpK2RHAkUsT\nmCRJUl/8gGqueK/HSwQySWuT04h4BlVievHse57EzOuc/gCT1PmwneZ2M9UfPZqdfWl+bKf58fdu\nfuxPc1uufelIdnxvn1rntJjWXHMaER+JiLUR8dyIeAVwGfAE0F34q079a0PTs53mZhvNj+00P7bT\n/NhO82M7zc02WkxtGjk9CLgE2Bd4ELgWeHlmPlw0KkmSJD2lNclpZnZKxyBJkqTZtea0viQ12x3A\nB4CtheOQpLJaM3K6APUU/Ydm2eVxqguHNbsmtlMu8LiYe5cF/cylbKNdiblpmtiXFsvDVO/VP7Lr\ndRxvp7uormIa2sXXG7fQ3xso0w/nine2/rSSfm921Ur6vduVPjybudpoqb47ZrPQn/lU3jN1qaK+\nicyleqOWt4h4E/CZ0nFIkiQVcGpmXlLiB5ucziAi9qW6m9QdNGHRL0kr3SDV+i3/J3DVIr3mu4DX\nAy9bpNeTtPKtBp4HbCk1adzkVJJ2QUS8CrgSeF1mfmHKc28CNgHHZua353id4+vXeQPwIqrbLO8H\nfBP448y8rWff44D/HTgG2B/4GfD/Av8lMx+v9/kU8J+ozgmOn9/LzNxtV+orSUvNa04laRdk5tcj\n4m7gVOALU54+FfjxXIlpjwDeA/wS+AgwQDX6uQk4tme//w14GrCR6mLVo4Gzgd+gSm4B/gp4FnBi\nHYcXUUpaFkxOJWnXbQKGI2KvzPwngIjYD/h3wAd38rX2AI7KzF/Wr/Nz4PyIODwzb6n3eWdm/kvP\nMX8bEbcB/1dEHJSZP83Mb9e3aD4xM3fhZiOS1F8uJSVJu+5iquu0Xt9T9kZgN3Z+YuX/GE9Ma9dQ\njXq+YLygNzGNiKfX18h/i+oz/SU7+fMkqVFMTiVpF2XmPwDfoTp9Pu5NwPWZeftOvtzdUx7/Y/3v\n/zJeEBHPjogLI+Jh4J+p1ov6OtX1pQM7+fMkqVFMThcgIt4WET+JiF9ExPUR4UzYHhFxbkRsn7Ld\nMveRK1tErImIzRFxT90m66fZ57yIuDciHouIv4+Ig0vEWtJc7RQRn5qmf325VLw9LgaOj4hnRcQL\ngZcDn17A6/xyhvIAiIhVwChVIrwnMEY1aepd9T69n+vHAKva2p8i4j0RcUNEPBoRD0TEZRFx6JR9\nmtqf+iYizoyImyJirN6ui4iTpuzjZ9Mc7WRfWjwmpzspIt4AfBQ4l+r02U3Alvr6Mk24mWoW8QH1\ndlzZcBphT+B7wFuZZlXliHgX8HbgDKoJLtuo+tav9jPIBpi1nWqXM7l/NeH2xJcC26lieRPwr8Bn\nl+DnHEk1i/9vqD6D1lIlqP9H7051f/rNOqa29qc1wCeokvQTgd2Br0TE06bs18T+1E93U/1xM0h1\nx4Yrgc0RcTj42dRj1naqtb0vLQonRO28YeCTmXkxVH9JAf8BOA3485KBNcyTmflg6SCaJDOvAK4A\niIjpZk6/A/hgZv5/9T6/DzwA/A5Lk+Q00jzaCeBfmta/MvPhiLgc+D2q60+vyMxHluBH/ZIqaf9W\nZm4FiIi3UJ3a396z3zuAa4FXU90qqnX9KTNP7n1ct9PPqBKLa3uealx/6qfM/NKUovdFxFlUSf0t\n+NkEzKudoOV9abE4croTImJ3qg+1r42XZbVQ7FeZvMyL4JD6tOxtEbEpIp5dOqAmi4jnU/2V3du3\nHgW+jX1rOq+qT9PeGhEbI+LXSgdUu5hqtPIQqhn8S+FW4Dbgo/Vp67cBX+zdoac/fZnqVP8ngN8G\nbqfd/WkfqsR+6h8NTe1PfRcRqyLijVSrRlztZ9P0prZTz1P2pUXgyOnO2Y9q9u0DU8ofoFo0W5Xr\ngbcA/wAcCLyf6kPuiMzcVjCuJjuA6ktzur51QP/DabTLgc8DPwFeCPw34MsRcWyWv6vIF6kmMAWw\neQHHzxT/U+WZ+WRE/DbwceDdVHew+wXwXarTjTDRny4BnkO1csCpVAMS319AXMtePQp/PnBtz5Jc\n0Oz+1DcRcQTVig+rgceAUzLztog4Fj+bnjJTO9VP25cWicmpFl1mbul5eHNE3ADcCZwCfKpMVFop\nMrP3NOIPI+IHVCOJr2Lxbvu5UNuBJ4EvZOa/7syBmfkNqj9+p5bfObW8Xh1gHUBE/GX9//WZeV9d\nduzErvmfgf9cl/8/O1WblWUjcDjwyt7ChvenfroVOIpqtYfXA5dGddcyTTZtO2XmjfalxeNp/Z3z\nENX1XvtPKd8fuL//4SwPmTkG/Aho3ezOnXA/1WibfWsnZeZPqH43m9C/Xkd1huXifvywiLgAOBl4\n1XhiWrM/9ZilnXbQsP7UN5n5ZGbeXidZ76U6bX8W9qVJZmmn6fZtZV9aDCanOyEznwBGgBPGy+pT\nRScA15WKq+ki4hlUv5yzfim0Wf0hdj+T+9beVBfa27dmEREHAftSsH9FxNER8UdUK3mMZua1Pc/t\nHhH7z7GtXsDPvAB4LfDqzLyr9zn704TZ2mmG/Yv3p4ZYBexmX5rTKqY54wH2pV3haf2d9zHgwogY\nAW6gmr3/dODCkkE1SUR8hOrauzup7vX9AeAJoNW3UIyIPamS9PEZ6C+IiKOARzLzbqrr4d4XET8G\n7qC67eVP2fF+7SvabO1Ub+dSXdd1f73fn1GNzG/Z8dX65iyqazpvBP5gynOvYPZTelkfM+/R1ojY\nSLVEzXpgW0SMj2qNZebj9f9b35/maqe6rzWxP/VVRHyY6nrJu4C9qPryWuBD9S6t70swezvZlxZX\neI3uzouItwLvpDqt8T3g7Mz8btmomiMiulTrC+5LtbzNtcB767/AW6u+fusqdpz0clFmnlbv836q\ntQT3obpt5dsy88f9jLO02dqJau3T/wm8mKqN7qX64D+nqcu3RMQA1Sofs/lhZk6dcDLba25n+slT\nfzC+zF293/tpcX+aq53qEetl1Z+WQkT8LfAaqgmsY1ST5v40M6/s2ef9tLgvweztZF9aXCanM4jq\nXtXrqP5KfHz2vSVJklaE1cDzgC2Z+XCJADytP7N1wGdKByFJklTAqVTL0fWdyenM7gDYtGkThx12\n2LQ7DA8Ps2HDhn7GtCzZTnOzjebHdpof22l+bKf5sZ3mtpLaaOvWrbz5zW+GOg8qoXXJaX03lT+h\nWjz4JqrrRb8zza6PAxx22GEMDg5O8zQMDAzM+Jwm2E5zs43mx3aaH9tpfmyn+bGd5rZC26jYJY2t\nWkoqIt5AtdTLucBLqJLTLRGxX9HAJEmSBLQsOaVa9umTmXlxZt4KnEl1+7HTyoYlSZIkaFFyGhG7\nUy3n8rXxsvpet18Fjp3pOEmSJPVPa5JTqlsK7gZMXUvwAarrT3dap9PZ1ZhawXaam200P7bT/NhO\n82M7zY/tNDfbaHG1Zp3TiDgQuAc4NjO/3VP+Z8DazDx2yv6DwMjatWsZGBiY9FqdTseOKEmSlrVu\nt0u3O/nmjWNjY1x99dUAQ5k5WiKuNiWnu1NdX/q7mbm5p/xCYCAzXzdl/0FgZGRkZCXOwJMkSdrB\n6OgoQ0NDUDA5bc1p/cx8AhgBThgvi4ioH19XKi5JkiRNaNs6px8DLoyIEeAGqtn7TwcuLBmUJEmS\nKq1KTjPzs/WapucB+wPfA9Zl5oNlI5MkSRK0LDkFyMyNwMbScUiSJGlHrbnmVJIkSc1ncipJkqTG\nMDmVJElSY7QqOY2INRGxOSLuiYjtEbG+dEySJEma0KrkFNiTaob+W4F23H1AkiRpGWnVbP3MvAK4\nAp5agF+SJEkN0raRU0mSJDWYyakkSZIaw+RUkiRJjdGqa04XYnh4mIGBgUllnU6HTqdTKCJJkqRd\n1+126Xa7k8rGxsYKRTMhMts5aT0itgO/k5mbZ3h+EBgZGRlhcHCwv8FJkiQVMDo6ytDQEMBQZo6W\niKFVI6cRsSdwMDA+U/8FEXEU8Ehm3l0uMkmSJEHLklPgpcBVVGucJvDRuvwi4LRSQUmSJKnSquQ0\nM7+Bk8AkSZIay0RNkiRJjWFyKkmSpMYwOZUkSVJjmJxKkiSpMVqTnEbEeyLihoh4NCIeiIjLIuLQ\n0nFJkiRpQmuSU2AN8AngGOBEYHfgKxHxtKJRSZIk6SmtWUoqM0/ufRwRbwF+BgwB15aISZIkSZO1\naeR0qn2oFuJ/pHQgkiRJqrQyOY2IAM4Hrs3MW0rHI0mSpEprTutPsRE4HHhl6UAkSZI0oXXJaURc\nAJwMrMnM++baf3h4mIGBgUllnU6HTqezRBFKkiQtvW63S7fbnVQ2NjZWKJoJkZmlY+ibOjF9LXB8\nZt4+x76DwMjIyAiDg4N9iU+SJKmk0dFRhoaGAIYyc7REDK0ZOY2IjUAHWA9si4j966fGMvPxcpFJ\nkiRpXJsmRJ0J7A18Hbi3ZzulYEySJEnq0ZqR08xsUyIuSZK0LJmwSZIkqTFMTiVJktQYJqeSJElq\nDJNTSZIkNUZrktOIODMiboqIsXq7LiJOKh2XJEmSJrQmOQXuBt4FDAJDwJXA5og4vGhUkiRJekqb\nlpL60pSi90XEWcAxwC0FQpIkSdIUrUlOe0XEKqrF9/cArikcjiRJkmqtSk4j4gjgW8Bq4DHglMz8\ncdmoJEmSNK5N15wC3AocBRwNXABcGhEvKRuSJEmSxrVq5DQznwRurx/eGBFHA2cBZ8x0zPDwMAMD\nA5PKOp0OnU5nyeKUJElaat1ul263O6lsbGysUDQTIjNLx1BMRHwNuCMz/3Ca5waBkZGREQYHB/sf\nnCRJUp+Njo4yNDQEMJSZoyViaM3IaUR8GLgcuAvYCzgVWAt8qGRckiRJmtCa5BT4deAi4EBgDPg+\nsC4zryoalSRJkp7SmuQ0M08vHYMkSZJm17bZ+pIkSWowk1NJkiQ1hsmpJEmSGsPkVJIkSY3R2uQ0\nIt4dEdsj4mOlY5EkSVKllclpRLyM6q5QN5WORZIkSRNal5xGxDOATcDpwM8LhyNJkqQerUtOgb8A\nvpiZV5YORJIkSZO1ZhF+gIh4I/Bi4KWlY5EkSdKOWpOcRsRBwPnAiZn5xHyPGx4eZmBgYFJZp9Oh\n0+kscoSSJEn90+126Xa7k8rGxsYKRTMhMrN0DH0REa8F/g74JRB18W5A1mV7ZE9jRMQgMDIyMsLg\n4GC/w5UkSeq70dFRhoaGAIYyc7REDK0ZOQW+Chw5pexCYCvwp9mWLF2SJKnBWpOcZuY24JbesojY\nBjycmVvLRCVJkqRebZyt38vRUkmSpAZpzcjpdDLzNaVjkCRJ0oS2j5xKkiSpQUxOJUmS1Bgmp5Ik\nSWqM1iSnEXFuRGyfst0y95GSJEnql7ZNiLoZOIGJRfifLBiLJEmSpmhbcvpkZj5YOghJkiRNrzWn\n9WuHRMQ9EXFbRGyKiGeXDkiSJEkT2pScXg+8BVgHnAk8H7g6IvYsGZQkSZImtOa0fmZu6Xl4c0Tc\nANwJnAJ8qkxUkiRJ6tWa5HSqzByLiB8BB8+23/DwMAMDA5PKOp0OnU5nKcOTJElaUt1ul263O6ls\nbGysUDQTIrOdt5ePiGcAdwHnZOYF0zw/CIyMjIwwODjY9/gkSZL6bXR0lKGhIYChzBwtEUNrrjmN\niI9ExNqIeG5EvAK4DHgC6M5x6Iym/rWh6dlOc7ON5sd2mh/baX5sp/mxneZmGy2u1iSnwEHAJcCt\nwKXAg8DLM/Phhb6gnXF+bKe52UbzYzvNj+00P7bT/NhOc7ONFldrrjnNTC8SlSRJarg2jZxKkiSp\n4UxOJanhPve5z7Fq1Sq+8IUv7PDcUUcdxapVq/jGN76xw3PPec5zOO644/oRoiQtmtac1l+A1QBb\nt26dcYexsTFGR4tMZFtWbKe52Ubz09Z22nvvvQH4/Oc/z7OfPXFju23btvHDH/6QX/mVX+Fzn/sc\ne+21F1C10+WXX85Pf/pTTjjhhFa22Xy0tT/tLNtpbiupjXryntWlYmjtUlJziYg3AZ8pHYckSVIB\np2bmJSV+sMnpDCJiX6pbnd4BPF42GknLyMFUK4IMA9fUZf8G2ES1Wsibe/b9BLAX1a2V5/IB4LeA\ntVTL4AH8MfBq4CLgnfX/x70L+F3gBOCfdr4aklpqNfA8YMuurGi0KzytP4P6DSnyF4Ok5SsibgT+\nCviN8QWsI2INsB04BPhRZv5zRATwb4G/ms9C1xGxGTgZ2CMzv12XvRD4GtV6zR8E/jUzb66fOwS4\nNTN3vBhVkmZ3Xckf7oQoSVpEWZ2O+iawpqd4DdWNPxJ4RV32YmBv4Np5vvS1QADHAUTEbsAxwLWZ\neTvwQM9zzwCO3InXlqTGMDmVpMV3DTAYEU+rHx8HfBm4iYmkdXw0dV4JZGZuBR6uXwuq5PbpTIxw\nXAe8sv7/K4Dd5vvaktQkJqeStPiuAXYHjo2IQ4Fn1mVXM5GcHgfckpk/34nXvQ54ef3/VwI/y8yf\n9Dz3yp7nEpNTScuQyekCRMTbIuInEfGLiLg+Il5WOqYmiYhzI2L7lO2W0nGVFhFrImJzRNxTt8n6\nafY5LyLujYjHIuLvI+LgErGWNFc7RcSnpulfXy4V7wy+SzWRci1VMvqzzPwxVYJ6dET8al1+zcwv\nMa1rgYGIOB84F9gvIh6IiMuAu4HnRsSBVMnpvcBpbe5PEfGeiLghIh4db6f6j4XefZZDf1pSEXFm\nRNwUEWP1dl1EnDRlHz+b5mgn+9LiMTndSRHxBuCjVF8ML6E6TbclIvYrGljz3AzsDxxQb64EDnsC\n3wPeSjWqNUlEvAt4O3AGcDSwjapv/Wo/g2yAWdupdjmT+1ejbk+cmU8ANzCRnI4nodcAewCnUsV/\n9U6+9PhI6In1vx+t/7878OfAv1DN2D+G6hKAtvenNVQrIhzDRDt9pedyi3GN7k99cDfV6g6DwBBw\nJbA5Ig4HP5t6zNpOtbb3pcWRmW47sQHXA/+953EAPwXeWTq2pmxUifto6TiavFFda7h+Stm9wHDP\n472BXwCnlI63Ye30KeDvSsc2j9g/SPUlfgdwdk/5D6mWlPol8KydfM3dgceoJlz9Enh5Xb5f3Vbf\n73nu5/anHdpvvJ2OW279qUBbPQz8Qf1/P5vm1072pUXaHDndCRGxO9VfS18bL8uqR34VOLZUXA11\nSH1a9raI2BQRz577kPaKiOdT/ZXd27ceBb6NfWs6r6pP094aERsj4tdKBzSNa4CnAc9m8un7q4FD\ngTsy896decGsRmS/Q9Un/gUYqZ/ah2qU+btM9Je9sT9NNd5Oj0wpXw79qS8iYlVEvJFqhP9qP5um\nN7Wdep6yLy0Ck9Odsx/VDNgHppQ/QPXLq8r1VIuKrwPOBJ5P9SG3Z8mgGu4Aqi9N+9bcLgd+H3gN\n1cLzxwNfrtcNbZLrqEYwH6W6/GfcNVTv9c6e0h93bX38dzPzibre59flm+vntmF/mqS3nTKz9xr4\n5dKfllSJQbepAAAgAElEQVREHBER/0T1R88nqUZFb8PPpklmaSewLy0aF+HXosvMLT0Pb46IG4A7\ngVOoTntIC5aZn+15+MOI+AFwG/Aq4KoiQU0jM/+Z6jT81PJL2IUbfGTme4H39hRtBA4HXpmZ9wG7\nRcSxOFN/qqfaqbdwufSnPrgVOAoYAF4PXBoRx5cNqZGmbafMvNG+tHgcOd05D1GNhOw/pXx/4P7+\nh7M8ZOYY8COq2zpqevdTXb9s39pJWS2l9BAt7F8RcQHVXaNeVSem4+xPPWZppx20tT9l5pOZeXud\nZL2X6rT9WdiXJpmlnabbt5V9aTGYnO6E+nqvEap7VQNPnSo6gcK3+mqyqO5WczAw65dCm9UfYvcz\nuW/tTTXL2L41i4g4CNiXZdq/ImLPiNh/jm2Hz+o64Xot8OrMvKv3OfvThNnaaYb9l3V/WkSrgN3s\nS3NaRXW53w7sSwvnaf2d9zHgwogYoVoqZpjqLi0XlgyqSSLiI8AXqU7l/wbwAeAJqvt/t1Z9ze3B\nVKMQAC+IiKOARzLzbqrr4d4XET+mmuH9QaqVIL5QINxiZmunejsX+DzVF+bBwJ9Rjcxv2fHVloU/\noarTTJLquu2nEquI2Ei1RM16YFtEjI9qjWXm4/X/W9+f5mqnuq+ttP600yLiw1TXS94F7EW11Nla\n4EP1Lq3vSzB7O9mXFpfJ6U7KzM/Wa5qeR3Va43vAusx8sGxkjXIQ1TV1+wIPUl379vLMfLhoVOW9\nlOq6o6y3j9blFwGnZeafR8TTqS6y34dq4sy/z8x/LRFsQbO101uB36SadLAP1RI3W4Bz6jMby9FF\nzL0Y/9TTp2dStc3Xp5T/AXAxgP0JmLudfsnK608L8etU/fBAYIxqSbJ1mXkV2Jd6zNhOEbEa+9Ki\niWolJC1XEbEv1az4O6juSCNJkpppNfA8YIsDNjNz5HT5Wwd8pnQQkiRp3k5lF1btWOlMThsoIt5G\ndR3aAVTrI56dmd+ZYfc7ADZt2sRhhx026Ynh4WE2bNiwhJE2l3VvZ92h3fW37ta9jZZT/bdu3cqb\n3/xmqL+7NT2T04aJiDdQXWN3BhMTrrZExKGZ+dA0hzwOcNhhhzE4ODjpiYGBgR3K2sK6t7Pu0O76\nW3fr3kbLtP5ehjcLl5JqnmHgk5l5cWbeSnVB/2PAaWXDkiRJWnompw0SEbsDQ0y+h3ECX6XF9zCW\nJEntYXLaLPtRLebrPYwlSVIrmZyuYJ1Op3QIxVj39mpz/a17O7W57mD9VyLXOW2Q+rT+Y8DvZubm\nnvILgYHMfN00xwwCI2vXrmVgYGDSc51Ox19aSZIK6Ha7dLuTb4w4NjbG1VdfDTCUmaNFAlsGTE4b\nJiKuB76dme+oHwfVrdI+npkfmWb/QWBkZGRkOc5WlCSpNUZHRxkaGgKT01m5lFTzfAy4MCJGmFhK\n6unAhSWDkiRJ6geT04bJzM9GxH7AecD+wPeo7t37YNnIJEmSlp7JaQNl5kZgY+k4JEmS+s3Z+pIk\nSWoMk9OGiYg1EbE5Iu6JiO0Rsb50TJIkSf1icto8e1JdZ/pWwKUUJElSq3jNacNk5hXAFfDUMlKS\nJEmt4cipJEmSGsPkVJIkSY1hcipJkqTG8JrTFWJ4eJiBgYFJZZ1Oh06nUygiSZLaq9vt0u12J5WN\njY0VimZ5iUwnhDdVRGwHficzN8+yzyAwMjIywuDgYP+CkyRJO2V0dJShoSGAocwcLR1PUzly2jAR\nsSdwMDA+U/8FEXEU8Ehm3l0uMkmSpKVncto8LwWuolrjNIGP1uUXAaeVCkqSJKkfTE4bJjO/gRPV\nJElSS5kESZIkqTFMTiVJktQYJqcNEhHviYgbIuLRiHggIi6LiENLxyVJktQvJqfNsgb4BHAMcCKw\nO/CViHha0agkSZL6xAlRDZKZJ/c+joi3AD8DhoBrS8QkSZLUT46cNts+VMtJPVI6EEmSpH4wOW2o\niAjgfODazLyldDySJEn94Gn95toIHA68snQgkiRJ/WJy2kARcQFwMrAmM++bzzHDw8MMDAxMKut0\nOnQ6nSWIUJIkzabb7dLtdieVjY2NFYpmeYnMLB2DetSJ6WuB4zPz9nnsPwiMjIyMMDg4uOTxSZKk\nhRkdHWVoaAhgKDNHS8fTVI6cNkhEbAQ6wHpgW0TsXz81lpmPl4tMkiSpP5wQ1SxnAnsDXwfu7dlO\nKRiTJElS3zhy2iCZ6R8LkiSp1UyGJEmS1Bgmp5IkSWoMk9MGiYgzI+KmiBirt+si4qTScUmSJPWL\nyWmz3A28CxgEhoArgc0RcXjRqCRJkvrECVENkplfmlL0vog4CzgG8BamkiRpxTM5baiIWEW1hNQe\nwDWFw5EkSeoLk9OGiYgjgG8Bq4HHgFMy88dlo5IkSeoPrzltnluBo4CjgQuASyPiJWVDkiRJ6g9H\nThsmM58Ebq8f3hgRRwNnAWfMdtzw8DADAwOTyjqdDp1OZ0nilCRJM+t2u3S73UllY2NjhaJZXiIz\nS8egWUTE14A7MvMPZ3h+EBgZGRlhcHCwv8FJkqR5Gx0dZWhoCGAoM0dLx9NUjpw2SER8GLgcuAvY\nCzgVWAt8qGRckiRJ/WJy2iy/DlwEHAiMAd8H1mXmVUWjkiRJ6hOT0wbJzNNLxyBJklSSs/UlSZLU\nGCanDRYR746I7RHxsdKxSJIk9YPJaUNFxMuolo+6qXQskiRJ/WJy2kAR8QxgE3A68PPC4UiSJPWN\nyWkz/QXwxcy8snQgkiRJ/eRs/YaJiDcCLwZeWjoWSZKkfjM5bZCIOAg4HzgxM58oHY8kSVK/mZw2\nyxDwTGA0IqIu2w1YGxFvB/bIGe43Ozw8zMDAwKSyTqdDp9NZynglSdI0ut0u3W53UtnY2FihaJaX\nmCHXUQERsSfw3CnFFwJbgT/NzK3THDMIjIyMjDA4OLj0QUqSpAUZHR1laGgIYCgzR0vH01SOnDZI\nZm4Dbukti4htwMPTJaaSJEkrjbP1m8+hbUmS1BqOnDZcZr6mdAySJEn94sipJEmSGsPkVJIkSY1h\nctogEXFuRGyfst0y95GSJEkrg9ecNs/NwAnA+DqnTxaMRZIkqa9MTpvnycx8sHQQkiRJJXhav3kO\niYh7IuK2iNgUEc8uHZAkSVK/mJw2y/XAW4B1wJnA84Gr6ztHSZIkrXie1m+QzNzS8/DmiLgBuBM4\nBfhUmagkSZL6x+S0wTJzLCJ+BBw8177Dw8MMDAxMKut0OnQ6naUKT5IkzaDb7dLtdieVjY2NFYpm\neYlM747ZVBHxDOAu4JzMvGCGfQaBkZGREQYHB/sanyRJmr/R0VGGhoYAhjJztHQ8TeU1pw0SER+J\niLUR8dyIeAVwGfAE0J3jUEmSpBXB0/rNchBwCbAv8CBwLfDyzHy4aFSSJEl9YnLaIJnpBaKSJKnV\nPK0vSZKkxjA5lSRJUmOYnDZMRDwrIj4dEQ9FxGMRcVM9I1+SJGnF85rTBomIfYBvAl+jukvUQ8Ah\nwD+WjEuSJKlfTE6b5d3AXZl5ek/ZnaWCkSRJ6jdP6zfLfwS+GxGfjYgHImI0Ik6f8yhJkqQVwuS0\nWV4AnAX8A/BbwF8CH4+I3ysalSRJUp94Wr9ZVgE3ZOZ/rR/fFBFHAGcCny4XliRJUn84ctos9wFb\np5RtBZ4z14HDw8OsX79+0nb22WcvSZDLQbfb3ju+trnu0O76W/d2anPdobn173a7O3wvDw8Plw5r\nWTA5bZZvAi+aUvYi5jEpasOGDWzevHnSdued7Z1L1dQPq35oc92h3fW37u3U5rpDc+vf6XR2+F7e\nsGFD6bCWBZPTZtkAvDwi3hMRL4yINwGnAxcUjkuSJKkvTE4bJDO/C7wO6AA/AN4LvCMzLy0amCRJ\nUp84IaphMvPLwJdLxyFJklSCyenytxpg69ap86hgbGyM0dHRvgfUBNa9nXWHdtffulv3NlpO9e/5\nrl5dMo6mi8wsHYN2QX1d6mdKxyFJkubt1My8pHQQTWVyusxFxL7AOuAO4PGy0UiawcHApcAwcE1d\n9m+ATcCtwJt79v0EsBfwlnm87h8DfwS8vv73FcA9U15PUnOsBp4HbMnMhwvH0lie1l/m6s7tX19S\ng0XEjcBfAb+RmaN12RpgO3AI8KPM/OeICODfAn81vt8cr3tv/d/zgB8B76QadFge5zildrqudABN\nZ3IqSUssMzMivgms6SleA1wGvJZqxPMrwIuBvYFrd/JH3JiZ3uZY0orgUlKS1B/XAIMR8bT68XFU\nK3PcxETSOj6aujPJaQKfXKwgJak0R04lqT+uAXYHjo2InwLPrMuOYCI5PQ64JTN/vpOv/ZNFi1KS\nCnPkVJL647tUkxbXUiWjP8vMH1MlqEdHxK/W5dfM/BIz+sWiRSlJhZmcrkAR8baI+ElE/CIiro+I\nl5WOqR8i4tyI2D5lu6V0XEshItZExOaIuKeu5/pp9jkvIu6NiMci4u8j4uASsS62ueoeEZ+aph8U\nv7FFZj4B3MBEcjqehF4D7AGcCuwPXD3Ta9S3Nr4hIh6NiAeAN86w34p776fWPSIui4hDp+zTyPd+\nMUTEmRFxU0SM1dt1EXHSlH1W3PsOc9d9Jb/vbWVyusJExBuAjwLnAi+hup5tS0TsVzSw/rmZ6gv+\ngHo7rmw4S2ZP4HvAW6muOZwkIt4FvB04Azga2EbVD361n0EukVnrXrucyf2g05/Q5nQNcAzwqvr/\n4ytu3Aq8i6o+s42crqFaauoY4ESqz/BV9CzovYLf+6l13x34Ss81vOOa+t7vqrup+sggMARcCWyO\niMNhRb/vMEfdayv1fW+nzHRbQRtwPfDfex4H8FPgnaVj60PdzwVGS8dRoN7bgfVTyu4Fhnse7011\n6veU0vH2oe6fAv6udGwzxPtbdcy/BF7cU/6XdfltO/l6f0aV0P77Fr73+9VtdtxyeO+XqA0eBv6g\nTe/7DHVv1fvehs2R0xUkInan+qvya+NlWf3mfhU4tlRcfXZIfbr3tojYFBHPLh1Qv0XE86lGDnr7\nwaPAt2lPP3hVfer31ojYGBG/Vjqg2nVUiemjVGc1xl1DlWTOeEp/BuMjpj+H1r33+1C12SNTypv6\n3i+aiFgVEW+kuhzk6ja971Pr3vPUin/f28TZ+ivLfsBuwANTyh8AXtT/cPrueqq76vwDcCDwfqoP\n7iMyc1vBuPrtAKov7en6wQH9D6fvLgc+TzWD/YXAfwO+HBHH1n+sFZOZ/0x1Onpq+SXs5M006gX7\nXwhcnZnfqotb8d7XdT8fuDYze68rb+x7vxgi4gjgW1R/lDxGNSp6W0Qcywp/32eqe/30in7f28jk\nVCtGZm7peXhzRNwA3AmcQnXaRy2QmZ/tefjDiPgBcBvVdZ5XFQlqaWwEDgdeWTqQAqatewve+1uB\no4ABqlvWXhoRx5cNqW+mrXtm3tiC9711PK2/sjxEdcpw/ynl+wP39z+csjJzjOqWjitixupOuJ/q\nWmP7AZCZP6H63Vg2/SAi9oyI/WfZ/m/gZOBVmXlfz6Er/r2PiAuYvu47WI7v/Wwy88nMvL1OyN5L\nddr+LFrwvs9S9+n2XVHvexuZnK4gWS1VMwKcMF5Wn/46gRbeyzcinkH14TTrF9hKU38w38/kfrA3\n1SznNvaDg4B9WV794E+o4p1uux84DTg1M+/qPWilv/d1Yvpa4NVT6z7D/svxvd8Zq4DdVvr7PoNV\nVJex7aAF7/uK52n9ledjwIURMUK1puIw8HTgwpJB9UNEfAT4ItWp/N8APgA8AXRLxrUUImJPqsQ7\n6qIXRMRRwCOZeTfV9Xjvi4gfA3cAH6RateELBcJdVLPVvd7Opbr+7P56vz+jGkHfsuOrNdZFTL+k\n1DuA1wDvA+6IiPGRsrHMfLz+/4p87yNiI9XyQOuBbVPrXveLlfDeTysiPkx1beVdwF5U6+KuBT5U\n77Ii33eYve4r/X1vrdLLBbgt/ka1/uMdVMuIfAt4aemY+lTvLtWH8S+oPsQuAZ5fOq4lquvxTCxJ\n1Lv9j5593k+1vMxjVB/SB5eOe6nrTjVZ4gqqL6nHgduplml6Zum4F6nu09X7l8DvT9lvxb33c9W9\nBe/939Z1+kVdx68Ar1np7/tcdV/p73tbt6jfXC1TEbEvsI4qGX189r0lSVJBq4HnAVuyugGHpuFp\n/QaKiLdRXXN2ANVaiGdn5ndm2H0d8Jl+xSZJknbZqezk8nFtYnLaMD23Hz2DiWtGt0TEoZn50DSH\n3AGwadMmDjvssElPDA8Ps2HDhqUNuKGsezvrDu2uv3W37m20nOq/detW3vzmN0P93a3pmZw2zzDw\nycy8GCAizgT+A9Xs3D+fZv/HAQ477DAGBwcnPTEwMLBDWVtY93bWHdpdf+tu3dtomdbfy/Bm4VJS\nDeLtRyVJUtuZnDbLbLcfXRG3oJMkSZqNyakkSZIaw2tOm2XBtx8dHh5mYGBgUtlzn/vcRQ1uOel0\nOqVDKKbNdYd219+6t1Ob6w7NrX+326XbnXwPmLGxsULRLC+uc9owEXE98O3MfEf9OKgWlP94Zn5k\nmv0HgZGRkZHleEG4JEmtMTo6ytDQEMBQZo6WjqepHDltntbeflSSJMnktGEy87MRsR9wHtXp/O8B\n6zLzwbKRSZIkLT2T0wbKzI3AxtJxSJIk9Zuz9SVJktQYJqeSJElqDJPThomINRGxOSLuiYjtEbG+\ndEySJEn9YnLaPHtSTYJ6K+A6X5IkqVWcENUwmXkFcAU8tcapJElSazhyKkmSpMYwOZUkSVJjmJxK\nkiSpMbzmdIUYHh5mYGBgUlmn06HT6RSKSJKk9up2u3S73UllY2NjhaJZXiLTCeFNFRHbgd/JzM2z\n7DMIjIyMjDA4ONi/4CRJ0k4ZHR1laGgIYCgzR0vH01SOnDZMROwJHAyMz9R/QUQcBTySmXeXi0yS\nJGnpmZw2z0uBq6jWOE3go3X5RcBppYKSJEnqB5PThsnMb+BENUmS1FImQZIkSWoMk9MGiYj3RMQN\nEfFoRDwQEZdFxKGl45IkSeoXk9NmWQN8AjgGOBHYHfhKRDytaFSSJEl94jWnDZKZJ/c+joi3AD8D\nhoBrS8QkSZLUT46cNts+VDP2HykdiCRJUj+YnDZURARwPnBtZt5SOh5JkqR+8LR+c20EDgdeWToQ\nSZKkfjE5baCIuAA4GViTmffN55jh4WEGBgYmlXU6HTqdzhJEKEmSZtPtdul2u5PKxsbGCkWzvERm\nlo5BPerE9LXA8Zl5+zz2HwRGRkZGGBwcXPL4JEnSwoyOjjI0NAQwlJmjpeNpKkdOGyQiNgIdYD2w\nLSL2r58ay8zHy0UmSZLUH06IapYzgb2BrwP39mynFIxJkiSpbxw5bZDM9I8FSZLUaiZDkiRJagyT\nU0mSJDWGyWmDRMSZEXFTRIzV23URcVLpuCRJkvrF5LRZ7gbeBQwCQ8CVwOaIOLxoVJIkSX3ihKgG\nycwvTSl6X0ScBRwDeAtTSZK04pmcNlRErKJaQmoP4JrC4UiSJPWFyWnDRMQRwLeA1cBjwCmZ+eOy\nUUmSJPWH15w2z63AUcDRwAXApRHxkrIhSZIk9Ycjpw2TmU8Ct9cPb4yIo4GzgDNmO254eJiBgYFJ\nZZ1Oh06nsyRxSpKkmXW7Xbrd7qSysbGxQtEsL5GZpWPQLCLia8AdmfmHMzw/CIyMjIwwODjY3+Ak\nSdK8jY6OMjQ0BDCUmaOl42kqR04bJCI+DFwO3AXsBZwKrAU+VDIuSZKkfjE5bZZfBy4CDgTGgO8D\n6zLzqqJRSZIk9YnJaYNk5umlY5AkSSrJ2fqSJElqDJNTSZIkNYbJaYNFxLsjYntEfKx0LJIkSf1g\nctpQEfEyqrVNbyodiyRJUr+YnDZQRDwD2AScDvy8cDiSJEl9Y3LaTH8BfDEzrywdiCRJUj+5lFTD\nRMQbgRcDLy0diyRJUr+ZnDZIRBwEnA+cmJlPlI5HkiSp30xOm2UIeCYwGhFRl+0GrI2ItwN7ZGZO\nd+Dw8DADAwOTyjqdDp1OZynjlSRJ0+h2u3S73UllY2NjhaJZXmKGXEcFRMSewHOnFF8IbAX+NDO3\nTnPMIDAyMjLC4ODg0gcpSZIWZHR0lKGhIYChzBwtHU9TOXLaIJm5DbiltywitgEPT5eYSpIkrTTO\n1m8+h7YlSVJrOHLacJn5mtIxSJIk9Ysjp5IkSWoMk9MGiYhzI2L7lO2WuY+UJElaGTyt3zw3AycA\n40tJPVkwFkmSpL4yOW2eJzPzwdJBSJIkleBp/eY5JCLuiYjbImJTRDy7dECSJEn9YnLaLNcDbwHW\nAWcCzweurhfnlyRJWvE8rd8gmbml5+HNEXEDcCdwCvCpMlFJkiT1j8lpg2XmWET8CDh4rn2Hh4cZ\nGBiYVNbpdOh0OksVniRJmkG326Xb7U4qGxsbKxTN8hKZ3oCoqSLiGcBdwDmZecEM+wwCIyMjIwwO\nDvY1PkmSNH+jo6MMDQ0BDGXmaOl4msprThskIj4SEWsj4rkR8QrgMuAJoDvHoZIkSSuCp/Wb5SDg\nEmBf4EHgWuDlmflw0agkSZL6xOS0QTLTC0QlSVKreVpfkiRJjWFyKkmSpMYwOW2YiHhWRHw6Ih6K\niMci4qZ6Rr4kSdKK5zWnDRIR+wDfBL5GdZeoh4BDgH8sGZckSVK/mJw2y7uBuzLz9J6yO0sFI0mS\n1G+e1m+W/wh8NyI+GxEPRMRoRJw+51GSJEkrhMlps7wAOAv4B+C3gL8EPh4Rv1c0KkmSpD7xtH6z\nrAJuyMz/Wj++KSKOAM4EPl0uLEmSpP5w5LRZ7gO2TinbCjxnrgOHh4dZv379pO3ss89ekiCXg263\nvXd8bXPdod31t+7t1Oa6Q3Pr3+12d/heHh4eLh3WsmBy2izfBF40pexFzGNS1IYNG9i8efOk7c47\n2zuXqqkfVv3Q5rpDu+tv3dupzXWH5ta/0+ns8L28YcOG0mEtCyanzbIBeHlEvCciXhgRbwJOBy4o\nHJckSVJfmJw2SGZ+F3gd0AF+ALwXeEdmXlo0MEmSpD5xQlTDZOaXgS+XjkOSJKkEk9PlbzXA1q1T\n51HB2NgYo6OjfQ+oCax7O+sO7a6/dbfubbSc6t/zXb26ZBxNF5lZOgbtgvq61M+UjkOSJM3bqZl5\nSekgmsrkdJmLiH2BdcAdwONlo5EkSbNYDTwP2JKZDxeOpbFMTiVJktQYztaXJElSY5icSpIkqTFM\nTiVJktQYJqeSJElqDJPTFSgi3hYRP4mIX0TE9RHxstIx9UNEnBsR26dst5SO6/9v797j5arqu49/\nvgmXGAgHASUCilyiQqk8JCpN1UCANlGfgheKRG5SkYp4S8tLHpEawAsqhVj0CbYPNZECsTyiFSzk\nIDcDcrMJUhGikoshYLgEOMSQKCS//rHWMXMmczlnJnNmn5nv+/Xar5OzZ+2ZtX97MfzOXmuv1QqS\n3i7pOkmP5fM8ukKZCyQ9LukFST+StH876rq11Tt3SXMrtIOOWNgiL218n6TnJT0h6fuSXlehXMdd\n+8Gce4df+49IekBSX97ukjS9rEzHXXeof+6dfN27lZPTDiPp/cDFwCzgEOABoFfSbm2t2PB5ENgd\nGJ+3t7W3Oi2zA/Az4KPAFlNuSDob+BhwOvAWYB2pHWw3nJVskZrnnt3IwHYwY3iq1nJvB74OHAoc\nBWwL3CTpZf0FOvja1z33rFOv/aPA2cBEYBJwK3CdpAOho6871Dn3rFOve1fyVFIdRtI9wL0R8cn8\nu0j/YV8aEV9ta+VaTNIs4JiImNjuugwnSZuAd0fEdSX7HgcuiojZ+fedgCeAUyLimvbUdOurcu5z\ngZ6IeG/7ajY88h+dTwJTIuLOvK9brn2lc++aaw8gaQ1wVkTM7Zbr3q/s3LvquncD3zntIJK2Jf1V\neUv/vkh/fdwMTG5XvYbZhNzdu1TSlZJe3e4KDTdJ+5DuHJS2g+eBe+mednB47vpdImmOpF3aXaEW\n2Zl09/gZ6LprP+DcS3T8tZc0StLxwPbAwm667uXnXvJSx1/3brJNuytgW9VuwGjSX8ulngBeP/zV\nGXb3AB8Efgm8CjiP9MV9UESsa2O9htt40v+0K7WD8cNfnWF3I3AtsBzYD7gQuEHS5OigrqLcK/I1\n4M6I6B9b3RXXvsq5Q4dfe0kHAXeTVhl6ATguIpZKmkyHX/dq555f7ujr3o2cnFrHiIjekl8flHQf\n8BvgOGBue2plw62sC/MXkn4OLAUOB25rS6VaYw5wIPDWdlekDSqeexdc+yXAwUAPcCzwHUmHtbdK\nw6biuUfE/V1w3buOu/U7y9PARtKg8FK7A6uHvzrtFRF9wK+AjnhidQhWA8LtAICIWE76b6Nj2oGk\nbwDvBA6PiN+WvNTx177GuW+h0659RLwUEctyQvZZUrf9GXTBda9x7pXKdtR170ZOTjtIRLwILAKO\n7N+Xu7+OBO5qV73aRdKOpC+nmv8D6zT5i3k1A9vBTqSnnLuxHewF7EqHtIOcnB0DTI2IlaWvdfq1\nr3XuVcp31LWvYBQwutOvexWjSMPYttAF173juVu/81wCzJO0CLgPmAmMBea1s1LDQdJFwPWkrvw9\ngfOBF4H57axXK0jagZR4K+/aV9LBwDMR8ShpPN65kh4BVgCfB1YBP2hDdbeqWueet1mk8Werc7mv\nkO6g9275biOLpDmkKXKOBtZJ6r9T1hcRG/K/O/La1zv33C46+dp/iTS2ciUwDjgBmAJ8IRfpyOsO\ntc+9069714oIbx22keZ/XAGsJw0gf1O76zRM5z2f9GW8nvQldjWwT7vr1aJzPQzYRBrGUbp9q6TM\necDjpIcHeoH9213vVp876WGJBaT/SW0AlgGXAa9od7230rlXOu+NwMll5Tru2tc79y649pfnc1qf\nz/Em4IhOv+71zr3Tr3u3bp7n1MzMzMwKw2NOzczMzKwwnJyamZmZWWE4OTUzMzOzwnByamZmZmaF\n4SWAOJgAACAASURBVOTUzMzMzArDyamZmZmZFYaTUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZ\nmZlZYTg5NTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJqZmZmZkVhpNTMzMzMysMJ6dmZoCkN0n6iaTf\nSdoo6Y3trpOZWTfapt0VMDNrN0nbAN8FXgA+lX/+pq2VahFJBwDHAXMjYmW762NmVs7JqZkZ7Ae8\nBvhQRMxtd2Va7EBgFnAb4OTUzArH3fpmZrB7/tlXr6CksS2uS6sJiHZXwsysGienZtbVJM0Fbicl\nbN+VtEnSrfm1eZLWStpX0g2SngeuLDn2UEkLJD0naZ2k2yX9eYXP2EPStyStlrRB0oOSTh1k/cZI\nulTSU5Kel/Qf+f02SfpcSbnXSJojaYmkFyQ9LekaSXuXlDkFuCb/ent+j42SppSUeYekhXns7fOS\nfijpwCEF1cysCe7WN7Nu901gFfBZ4J+AnwJP5NeC9D3ZC9wB/D1pPCqSjgBuAP4LOA/YBJwK3Crp\nbRHxX7ncK4F7gY3ApcDTwDuAf5U0LiIurVO/bwPHAlfk9zkM+E+2vPv5ZuDPgPn5fF4LfBS4TdKB\nEbEB+HGuw8eBLwBL8rEP57qeBMwDFgCfBsYCZwB3SDrEY1TNbDgowr07ZtbdJB1GGoN5bER8r2T/\nXOBk4MKIOLfsmF8Cj0TEu0r2bQ88BPw6IqbnfZcD04GDIuK5krJX5/2viojfV6nXIcAi4JKIOKtk\n/7eAU4DzI+KC/s8ufx9JbwHuAU6KiKvyvveR7p5OjYiFJWV3AB4F/j0izijZ/wrgV3n/R2qE0cxs\nq3C3vplZfd8s/UXS/wImAPMl7dq/AeOAW4ApJcXfC1wPjC4rexPQA0ys8bnTSXdILyvb/3XS2NE/\nKk1MJW0jaRdgGfBcnc/o9xe5Pt8pq2eQ7thOHcR7mJk1zd36Zma1vRQRq8r2Tcg/r6hyzCZJPcB2\nwM7A6cDfVigXwCtrfPbepOECy8v2P1JeUNIY4Bzgg8CebE5eg5R01jMhH3NblXrWfVjMzGxrcHJq\nZlZbpS73/l6nvwceqHLc74Dd8r+vJI0dreS/G6/aAN8gdfXPJnXl95GSyn9ncL1ko3L5E9k85rbU\nS1unmmZmtTk5NTMbuqX559qIuLVaIUlPAWuB0bXK1fAbUtK4T8lnwuY7t6XeB8yLiE+XfP72pDu3\npao9aLCUdOf0qQbrama2VXjMqZnZ0C0iJXNn5QeJBpC0G0BEbAKuBd4n6U+qlauhl5QwfrRs/8fZ\nMsncyJbf6Z8ARpftW5ffszxp7QWeB87JK2YNta5mZluF75yamQ1RRISk00hTSf0iP9X/GGms51RS\nl/oxufj/AQ4H7pX0/0hP8+8CTAKOYHPXf6XPWSzpWuBTOTm8hzSVVP+d09IE9YfASXku1oeAycCR\npKmrSv2MlMieLWln0rCFWyLiaUlnkMbRLpb0HeAp0spZ7wLuJCW7ZmYt5eTUzCyp1t1dcX9E/FjS\nZOAfgDOBHYHVpCfb/7mk3JN5SqfPAe8hzRu6BvgFaS7Rek4CfgvMyMffAhwP/BLYUFLuE6RxoR8A\nxpCSyaNId0T/eA4R8YSkvwU+A1xOurM6FVgYEfMlPUZKqM8Cticl3XcAnb6sq5kVhOc5NTMbYfJU\nVouBEyJifrvrY2a2NXnMqZlZgeUposp9itQ1v7DCa2ZmI5q79c3Miu3TkiaR5h99CXgnMA3454h4\nrK01MzNrgZbdOZX0cklXSeqT9Kykyys91Vrj+G9K2iTJA/DNrJvdBbwcOBf4R2B/YBbwsXZWysys\nVVp55/RqYHfS06LbAfNIDwmcWO9ASe8BDiUNxDcz61oRcTNwc7vrYWY2XFpy51TSG0jdTh+KiP+K\niLtI8/IdL2l8nWP3BP6J9MSpVyQxMzMz6yKtunM6GXiONEH1/yatDf090nQmhwI/KD8gT/r8RdJk\n09uR7hTslLeqJO1KSoRXMHBaFTMzMzMrhjHAa4HeiFhTq2CrktPxpATzAAZ267+UX6tkLGkOv18D\n7yeNsVoIfBj4Qo3PmgZctTUqbWZmZmYtdQJp6GdVQ0pOJV0InF2jSJAS0lcAO5C69e/Px34cWED1\nO6ETgHHAlIhYnY9ZA7xa0l4RsarKcSsArrzySg444ICKBWbOnMns2bNrVNtqcfwa59g1x/FrnGPX\nHMevcY5dczo1fg8//DAnnngi5LytlqHeOf1H6q8SsgzoIa3wd7+kM0krjfTfMa2cQcLbSEntKknl\na0HfAexT5bgNAP/yL/9CT0/PgBdmzJjBjBkz6OnpYeLEiXWqbdU4fo1z7Jrj+DXOsWuO49c4x645\nnRC/+fPnM3/+wPVB+vr6+v9ZdwjmkJLTPEag5jgBAEl96YfOBs4HTge2JS2Vd7ykT0dE+XrPV5CW\n8/sB8J3883ukpfXOr/eZs2fPHvEX08zMzGyk6785WGrx4sVMmjRpUMcP6Wl9SRfmuUerbRslvQ54\nClhHWnP6+8BS0hrSG4DfA3+T32+JpGMAIuJZ4C9y2b8FTiOtgPIz4Lih1NPMzMzMRqZWdeuvBv4A\n7AwcDUwHrgVOIa0HPTmXnUAaAtDvz4BbgP8PvBr4Lelu6rvqVWzmzJlVu/XNzMzMbHjU6davq1Xd\n+nezOemcAxwLnJQ/bw0p8SQiyseW7g9MJE03sB54EHgYmCFp+4j4fbXPrNWt7wS1OY5f4xy75jh+\njXPsmuP4Nc6xa04nxK/Zbn1FRCvqhaRbgCNI00d9mbQy1IukxPTBiHizpCXA2RHxgzzP6Qu5/KnA\nctJE/GeShh+MrZScSpoILFq0aJHHnJqZmZkVUElyOikiFtcq28rlS99PGnsKaQ3o7wKfzPtelveX\nduvvSXpoahs2z3+lvP2h1l1TcLe+mZmZWRE0263fyjun25IefnqQNH/peOAB4GBgaUQcVOGYLwPv\niIiDJb0VuJ101/ThSuXzMb5zamZmZlZgQ7lzOqSn9Ydot/zzT4GbgPeSJuAfQ+re73/6/9slx3wT\n2FfSbNLd0zW5jp9rpiLl2bsNjePXOMeuOY5f4xy75jh+jXPsmuP4tTY5hdQlv4i0xOj3gD7SuNJd\n8uvjyQ9HAUTECtKT+acAewGvBJ6IiO/V+6CZM2dy9NFHD9j6L7AvdHMcv8Y5ds1x/Brn2DXH8Wuc\nY9ecTojf/Pnzt8jJZs6cOejjWznm9Ln8c0FEnNu/U9IjwFiAiDi1wnH7ke6YbkeaI/VPBvNhnoTf\nzMzMrP2GdRL+Ido5/9y/f4ckke6GvljpAEkTgEtID0cdDzzSwvqZmZmZWcG0uls/gHdLOlnSG0hj\nSrcFnoSBY04ljQJuJD089XfAT4EdgW0k7dTiepqZmZlZAbSyW/9p0vKjc4ELgN1JS5HeTJrLFAaO\nOR1H6tIHuCxv/Z6VdGRE3F7hc8YAnHbaaYwbN27AC9OmTWP69On09fWxeHHNB8OsBsevcY5dcxy/\nxjl2zXH8GufYNacT4rdgwQJ6e3sH7Fu7dm3/P8fUO75lU0kBSLoHuDciPpl/F7ASuDQiLiorK+CA\nsrc4E5gKvA9YERHrK3zGB4CrWlB9MzMzM9u6ToiIq2sVaOWdU0jjR+dJWgTcB8wkPQw1D1K3PrBH\nRJwSKUt+qPRgSU8CGyLi4Rqf0QucAKwANmztEzAzMzOzpo0BXkvK22pqaXIaEddI2o2B3frTIqJ/\n5agBU0k1+Blr2LyilJmZmZkV012DKdTSbn0zMzMzs6Fo9dP6ZmZmZmaD5uTUzMzMzArDyamZmZmZ\nFUbHJ6eSzpS0XNJ6SfdIenO761R0kmZJ2lS2PVT/yO4k6e2SrpP0WI7V0RXKXCDpcUkvSPqRpP0r\nvVe3qRc7SXMrtMUb2lXfIpH0GUn3SXpe0hOSvi/pdRXKue1VMJj4uf1VJukjkh6Q1Je3uyRNLyvj\ndldFvfi53XV4cirp/cDFwCzgEOABoDfPIGC1PUiaYWF83t7W3uoU2g6kmSg+SloVbQBJZwMfA04H\n3gKsI7XD7YazkgVVM3bZjQxsizOqlOs2bwe+DhwKHEVafe8mSS/rL+C2V1Pd+GVuf1t6FDgbmAhM\nAm4FrpN0ILjdDULN+GVd3e46+mn9KosAPEpaBOCrba1cgUmaBRwTERPbXZeRRtIm4N0RcV3JvseB\niyJidv59J+AJ4JSIuKY9NS2eKrGbC/RExHvbV7ORIf/R/SQwJSLuzPvc9gapSvzc/gZJ0hrgrIiY\n63Y3dGXx6/p217F3TiVtS/qL5Jb+fXmi/5uBye2q1wgyIXe1LpV0paSm5qPtVpL2If3VW9oOnwfu\nxe1wsA7P3a5LJM2RtEu7K1RQO5PuPj8DbnsNGBC/Em5/NUgaJel4YHtgodvd0JTHr+Slrm53rV4h\nqp12A0aT/lor9QTw+uGvzohyD/BB4JfAq4DzSF86B0XEujbWayQaT/ofXqV2OH74qzPi3AhcCywH\n9gMuBG6QNDk6udtniHKv0NeAOyOif3y4294gVYkfuP1VJekg4G7Sqj8vAMdFxFJJk3G7q6ta/PLL\nXd/uOjk5tQZFROnSYg9Kug/4DXAcMLc9tbJuVNYF+AtJPweWAocDt7WlUsU0BzgQeGu7KzJCVYyf\n219NS4CDgR7gWOA7kg5rb5VGlIrxi4j73e46uFsfeBrYSBpQXGp3YPXwV2fkiog+4FeAn7YcutWA\ncDvcKiJiOem/bbfFTNI3gHcCh0fEb0tectsbhBrx24Lb32YR8VJELMvJ1GdJ3fZn4HY3KDXiV6ls\n17W7jk1OI+JFYBFwZP++3HVzJINc29USSTuS/qOo+cVtW8pfKqsZ2A53Ij0h7HY4RJL2AnbFbRH4\nY2J1DDA1IlaWvua2V1+t+FUp7/ZX3ShgtNtdw0aRhiJuoRvbXad3618CzJO0CLgPmAmMBea1s1JF\nJ+ki4HpSV/6ewPnAi8D8dtarqCTtQErelXftK+lg4JmIeJQ0lu1cSY8AK4DPA6uAH7ShuoVSK3Z5\nm0Uae7U6l/sK6S5+75bv1l0kzSFNL3M0sE5S/52qvojYkP/ttldFvfjltun2V4GkL5HGRa4ExgEn\nAFOAL+Qibnc11Iqf210WER29keZPXAGsJw0+flO761T0jZSErsoxWwlcDezT7noVdQMOAzaRhpGU\nbt8qKXMe8Dhp4HsvsH+7612ErVbsSA8KLCB9QW8AlgGXAa9od72LsFWJ20bg5LJybnsNxM/tr2bs\nLs/xWJ/jcxNwRFkZt7sG4ud2l7aOnufUzMzMzEaWjh1zamZmZmYjj5NTMzMzMysMJ6dmZmZmVhhO\nTs3MzMysMJycmpmZmVlhODk1MzMzs8JwcmpmZmZmheHk1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwn\np2ZmZmZWGE5OzczMzKwwnJyamZmZWWE4OTUzMzOzwnByamZmZmaF4eTUzMzMzArDyamZmZmZFYaT\nUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZmZlZYTg5NTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJ\nqZmZmZkVhpNTMzMzMysMJ6dmZmZmVhhOTs3MzMysMJycmpmZmVlhODk1MzMzs8JwcmpmZmZmheHk\n1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwnp2ZmZmZWGE5OzczMzKwwnJyamZmZWWE4OTUzMzOzwnBy\namZmZmaF4eTUzMzMzArDyamZmZmZFYaTUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZmZlZYTg5\nNTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJqZmZmZkVhpNTMzMzMysMJ6dmZmZmVhhOTs3MzMysMJyc\nmplVIOkwSZskTSnZN0/S8nbWy8ys0zk5NTOrLir8Xr6vI0iaIemT7a6HmZmTUzMzA/gA4OTUzNrO\nyamZmZmZFYaTUzPrKpJeI2mOpCWSXpD0tKRrJO3d4PuNlXSxpJWSNuT3/fsqZU+UdK+kdZKekfRj\nSUc1+LlvzMe/IOlRSZ+VdGoeJ/uaknJHS/qhpMdy/R6RdK6kUSVlbgPeBeydj98kaVnJ69tJOl/S\nr/N7rJT0FUnbNVJ3M7Natml3BczMhtmbgT8D5gOrgNcCHwVuk3RgRGwY4vtdDxwGXA48AEwDLpK0\nR0T8MUmVNAuYBfwE+AfgD8ChwBHAzUP5QEl7ALcBG4EvAi8Ap+X3LB8T+0FgLXAx8Lv8eRcA44Cz\nc5kvAD3AnsCnAOWySFI+xz8H/hlYAvwpMBOYALx3KHU3M6tHER05tt/MrCJJ20fE78v2vQW4Bzgp\nIq7K+w4DbgWmRsTCvG8ucFhE7Jt/Pwb4PnBORHy55P2uISVtEyJiuaT9gF8C34+Iv94K53ApKaE+\nJCJ+nvftDDwCvBzYJyJW1jjfy4ATgV0i4sW873rgT/rPraTsicBcYEpE3F2y/3TgMuCtEXFPs+dk\nZtbP3fpm1lVKEzVJ20jaBVgGPAdMHOLbvQN4Cfh62f6LSd+v78i/v4d0N/KCRupcwTTg7v7EFCAi\nngOuKi9Ydr47StoVuBMYC7xhEJ91LPAw8CtJu/ZvpDu3AqY2dSZmZmXcrW9mXUXSGOAcUnf3nqQE\nC1J3eM8Q325v4PGIWFe2/+GS1wH2BTaV7G/W3sBdFfY/Ur5D0oGkrv+pwE4lLw32fCeQktinKrwW\nwCsH8R5mZoPm5NTMus03gFOA2aSu/D5SkvXvdFhvkqQeYCHprvC5pDvEG4BJwJcZ3PmOAn5OGmOq\nCq8/ulUqa2aWOTk1s27zPmBeRHy6f4ek7YGdG3iv3wBHStqh7O7pAfnnivxzKSnJOxD47wY+p9Ln\n7l9h/4Sy3w8njUE9JiJ+0r8zj4EtV+0BhKXAGyPitgbqaWY2ZB11l8DMbBA2suV33yeA0Q281w2k\nP/I/VrZ/Jqkbf0H+/T9Iyd/n8tPvFUkaL+n1kurVpReYLOmNJcfuQppIv9RG0t3O0mmjtiM9TFVu\nHZW7+a8B9pL04Qr1HSNpbJ26mpkNie+cmlm3+SFwkqTngYeAycCRwNMVylZNJLPrSQ8GfVHSPmye\nSuqvgNkRsRwgIpZK+iKpa/0OSd8Dfk+a1uqxiPhsfr8vAyeTprdaWeNzv0p62v5mSV8nJZanke6o\nvpzNd0HvAp4FrshP+JOPq3SXdBFwnKSLgZ8Cv4uIHwL/BhwHXCZpKmkqrNGku8N/DfwlsLhOnMzM\nBs3JqZl1m0+QnrD/ADCG9OT6UaS7keVJW6Uk7o/7IiIk/RXpKfz3kx6yWgGcFRGzBxwUMStPbP9x\n0ryiL5C6+K8oe+9N9U4gIlZJOhy4FPgMKbG+jDQ36ddI40qJiGckvYs0e8DnSYnqv5GmyOote9s5\nwMH5HD5FSnR/mM/xGNLd4JOBd+e6LyON2/1VvfqamQ2F5zk1M+sQkr4GfBjYMfzlbmYjlMecmpmN\nQHlKrNLfdyV12d/hxNTMRrKWJaeSXi7pKkl9kp6VdLmkHYZw/Dfz+s6faFUdzcxGsLslzZZ0uqTP\nkcaMjiN135uZjVitHHN6NbA76UGD7YB5pHWZT6x3oKT3kNacfqyF9TMzG8n+k7R604dJY1UXAaeW\nThllZjYStWTMqaQ3kJ6CnRQR9+d900hfpntFxOoax+4J3E164vUG0hOvl1Yrb2ZmZmado1Xd+pNJ\nK5Kc1d+tT5qKJEh3RLeQ17j+CrCEdMf1ZmA3Bi63Z2ZmZmYdrFXd+uNJXfkHMLBb/6X8WiVjgfcA\nvyZNyfJy0rJ7HyZNu1JRfghgGmn6lg1bo/JmZmZmtlWNIc3h3BsRa2oVHFJyKulC4OwaRYKUkL4C\n2AH4UEm3/sdJq6VUuxM6gTSYf0p/t7+kNcCrJe0VEauqHDcNuGoo52FmZmZmbXEC6bmkqoZ65/Qf\ngbl1yiwjLYEX/Ylpdmv+WW396reRktpHS1b3G01aoeUOYJ8qx60AOOSQQxg3btyAF6ZNm8b06dOZ\nOXMms2fPrnSsDYLj1zjHrjmOX+Mcu+Y4fo1z7JrTCfFbsGABvb0D1/lYu3Yt999/P+S8rZYhJaf5\nNmzNW7EAkvrSDx0C/DlwFrBHfnnHKoddAfyItJzfp4H9c/1WkZbHq2YDwOWXX87EiRMrFujp6an6\nmtXn+DXOsWuO49c4x645jl/jHLvmdEL8Jk6cyDnnnDNg3+LFi5k0aRIMYgjmkB6IknRhnnu02rZR\n0uuAp0hrPX8XuAS4Engc2AicJmm3/H5L8rJ4RMSzpCXx/i/pKf07SWNU9yCNUTAzMzOzDteqbv3V\nwB/YPL70Y8C1wCnAeuBvgK+Sxpn2lBx7Rj5+f2AX0jyna0hrOv+o1ofOnDmTnp6eAftmzJjBjBkz\n6p2TmZmZmW0l8+fPZ/78+QP29fX1Dfr4VnXr301KOjcB/0p6aOmk/Hn3k6aaIiJGlx36LtLY0gNI\nSeyDwE+A0+p95uzZs0f8bXAzMzOzka7SzcGSbv26WjLPaUQsAW4nJaMfYnO3/q9JDz7tBQO79SVt\nA7yO9ADUicBRwH3Ap4CdJG3faH1897Q5jl/jHLvmOH6Nc+ya4/g1zrFrjuPXohWiACS9njSh/ots\nHn/6SVKX/7MRsbekjaTl9q6QtDfpCa7IG6REVfn3l0XE7yt8zkRg0ZQpU9ytb2ZmZtZm1br1Fy5c\nCGn10MW1jm9lcvoy0gNOC4A3kCbffwDYExgdEXtUOObHwKKI+DtJbyXdfRXwh4gYW+VzJgKLFi1a\n5G59MzMzswIq6davm5y2avlS2Dyf6V8Cs4BDSMnpXqSn9iu5GzhSUg/wbWBpruNPmqlIefZuQ+P4\nNc6xa47j1zjHrjmOX+Mcu+Y4fq1NTvtVvTWbp6b6dsmubwL7kpLUZcB+pDunX2mmAr7QzXH8GufY\nNcfxa5xj1xzHr3GOXXMcv6FPJTUUz+WfNwEXALsDPyNNqt//lP544NX9B0TECknfIE0ddQCwFngm\nIm6u92GeSsrMzMys/YZ1Kqkh6u/Wfz4iXgtpySigD3gWICJOLT1A0gTgI6QpqP4KmAQcM5gP81RS\nZmZmZu1XyKmkSgTwbkknS3oDqdt+W+BJGNitL2kUcCMwDvg74KekpU63kbRTpTc3MzMzs87Syjun\nT5MefJrLwG79m0nLksLAbv1xpDGmAJflrd+zko6MiNsrfM4YgNNOO41x48YNeGHatGlMnz6dvr4+\nFi+u+WCY1eD4Nc6xa47j1zjHrjmOX+Mcu+Z0QvwWLFhAb2/vgH1r167t/+eYese3bCopAEn3APdG\nxCfz7wJWApdGxEVlZUUaZ1rqTGAq8D5gRUSsr/AZHwCuakH1zczMzGzrOiEirq5VoJV3TgEuAeZJ\nWkRa7WkmMBaYB6lbH9gjIk6JlCU/VHqwpCeBDRHxcI3P6AVOIE3gv2Frn4CZmZmZNW0M8FpS3lZT\nS5PTiLhG0m4M7NafFhFP5SIDntZv8DPWADUzcDMzMzNru7sGU6il3fpmZmZmZkMxHJPwm5mZmZkN\nipNTMzMzMysMJ6dmZmZmVhgdn5xKOlPScknrJd0j6c3trlPRSZolaVPZ9lD9I7uTpLdLuk7SYzlW\nR1coc4GkxyW9IOlHkvZvR12Lpl7sJM2t0BZvaFd9i0TSZyTdJ+l5SU9I+r6k11Uo57ZXwWDi5/ZX\nmaSPSHpAUl/e7pI0vayM210V9eLndtfhyamk9wMXA7OAQ4AHgN48g4DV9iBphoXxeXtbe6tTaDuQ\nZqL4KGlVtAEknQ18DDgdeAuwjtQOtxvOShZUzdhlNzKwLc6oUq7bvB34OnAocBRp9b2bJL2sv4Db\nXk1145e5/W3pUeBsYCJpmfFbgeskHQhud4NQM35ZV7e7jn5av8oiAI+SFgH4alsrV2CSZgHHRMTE\ndtdlpJG0CXh3RFxXsu9x4KKImJ1/3wl4AjglIq5pT02Lp0rs5gI9EfHe9tVsZMh/dD8JTImIO/M+\nt71BqhI/t79BkrQGOCsi5rrdDV1Z/Lq+3XXsnVNJ25L+Irmlf1+e6P9mYHK76jWCTMhdrUslXSmp\nqflou5WkfUh/9Za2w+eBe3E7HKzDc7frEklzJO3S7goV1M6ku8/PgNteAwbEr4TbXw2SRkk6Htge\nWOh2NzTl8St5qavbXatXiGqn3YDRpL/WSj0BvH74qzOi3AN8EPgl8CrgPNKXzkERsa6N9RqJxpP+\nh1epHY4f/uqMODcC1wLLgf2AC4EbJE2OTu72GaLcK/Q14M6I6B8f7rY3SFXiB25/VUk6CLibtOrP\nC8BxEbFU0mTc7uqqFr/8cte3u05OTq1BEVG6tNiDku4DfgMcB8xtT62sG5V1Af5C0s+BpcDhwG1t\nqVQxzQEOBN7a7oqMUBXj5/ZX0xLgYKAHOBb4jqTD2lulEaVi/CLifre7Du7WB54GNpIGFJfaHVg9\n/NUZuSKiD/gV4Kcth241INwOt4qIWE76b9ttMZP0DeCdwOER8duSl9z2BqFG/Lbg9rdZRLwUEcty\nMvVZUrf9GbjdDUqN+FUq23XtrmOT04h4EVgEHNm/L3fdHMkg13a1RNKOpP8oan5x25byl8pqBrbD\nnUhPCLsdDpGkvYBdcVsE/phYHQNMjYiVpa+57dVXK35Vyrv9VTcKGO1217BRpKGIW+jGdtfp3fqX\nAPMkLQLuA2YCY4F57axU0Um6CLie1JW/J3A+8CIwv531KipJO5CSd+Vd+0o6GHgmIh4ljWU7V9Ij\nwArg88Aq4AdtqG6h1Ipd3maRxl6tzuW+QrqL37vlu3UXSXNI08scDayT1H+nqi8iNuR/u+1VUS9+\nuW26/VUg6UukcZErgXHACcAU4Au5iNtdDbXi53aXRURHb6T5E1cA60mDj9/U7joVfSMloatyzFYC\nVwP7tLteRd2Aw4BNpGEkpdu3SsqcBzxOGvjeC+zf7noXYasVO9KDAgtIX9AbgGXAZcAr2l3vSQsY\nWQAAAJpJREFUImxV4rYROLmsnNteA/Fz+6sZu8tzPNbn+NwEHFFWxu2ugfi53aWto+c5NTMzM7OR\npWPHnJqZmZnZyOPk1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwnp2ZmZmZWGE5OzczMzKwwnJyamZmZ\nWWE4OTUzMzOzwnByamZmZmaF4eTUzMzMzArDyamZmZmZFYaTUzMzMzMrjP8BAxS2H/AO2kQAAAAA\nSUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", - "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", - "\n", - "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", - "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", - "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", - "_y = y[0] ; _X = X[0]\n", - "\n", - "plt.figure(figsize=[8,8])\n", - "\n", - "plt.subplot(611) ; plt.title('y')\n", - "plt.imshow(_y.T, interpolation='none')\n", - "plt.subplot(612) ; plt.title('y_hat')\n", - "plt.imshow(_y_hat[0,:,:].T, interpolation='none')\n", - "plt.subplot(613) ; plt.title('w_W')\n", - "plt.imshow(_w_w[0,:,:].T, interpolation='none')\n", - "plt.subplot(614) ; plt.title('w_r')\n", - "plt.imshow(_w_r[0,:,:,0].T, interpolation='none')\n", - "plt.subplot(615) ; plt.title('free gate')\n", - "plt.imshow(_f[0,:,:].T, interpolation='none')\n", - "plt.subplot(616) ; plt.title('alloc. gate')\n", - "plt.imshow(_g_a[0,:,:].T, interpolation='none')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/repeat-copy/.ipynb_checkpoints/repeat-copy-nn-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/repeat-copy-nn-checkpoint.ipynb new file mode 100644 index 0000000..30a41bc --- /dev/null +++ b/repeat-copy/.ipynb_checkpoints/repeat-copy-nn-checkpoint.ipynb @@ -0,0 +1,404 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Using a Differentiable Memory Architecture (DNC)\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from nn_controller import NNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 10000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"nn_models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAADzxJREFUeJzt3W2sHNV5wPH/cw3YCQSj4hSsEgEpSRuuEQ04pDThpcHI\niKrQShXBRaqARAhI+sFVW4KMREoJpI0ItFBHfTNORbCbqolK1IDBQJsiQmnshMbXUBReAi02xSAc\nydiu7Xv6Yfa292X33j17Z3Z3dv8/aT/s7Ozsc+Y5O2dnZ+aZSCkhSZLqa6TXAUiSpPlxMJckqeYc\nzCVJqjkHc0mSas7BXJKkmnMwlySp5hzMJUmqOQdzSZJqzsFckqSaO6zKhUfEscBK4GVgX5WfJUnS\ngFkEnARsSim9OduMHQ3mEfEZ4HeB44FngN9OKf1bk1lXAl/r5DMkSRIAVwD3zzZD9mAeEZ8E7gCu\nAZ4GVgObIuKDKaVd02Z/GeDS+36VJR9aMuWFR1Zv5sI7V+R+/EyZteVHlq/Pmn/8e1eWsvxNFL9s\nZix/y1VZy69abqn+dctbdaHmLb76ewdLWHZzn9rS/rKB7MYuaJHbh4CLmi0+L5q+6wuttPru5ved\nwzM/Oe8DsvtDEy23U3223WnVN3P1qi+Xt91prtV2p1V+8/tmVXYB34DGWDqbTvbMVwN/nlL6G4CI\nuBb4FeBq4I+nzbsPYMmHlrD0jOOnvLBw8cIZ0zqS+6XKXPx4Zoytlr8IWFrC8quWf9+dVp2+eYuX\nnnGghGU3l7ds8gfzFtNb5TZ7A9hnfaGVVt/d/L5zROb8eR+Q3R+aaLmd6rPtTqu+matXfbm87U5z\nrfrComNajUO5fbNycx6mzupjEXE4cCbw6MS0VNx2bTNwdm50kiRp/nJ/MC6h+BH4+rTpr1McP5ck\nSV3mpWmSJNVc7jHzXcAh4Lhp048DdrZ60yOrN7Nw8cIp0xafeHTmR9fbsl4H0HXD0+LhaWlhdNWp\nvQ6ha4aprTCEffnyfsrvD4Ft06a1f0V31mCeUjoQEVuAC4AHACIiGs//tNX7LrxzRTknu9XYsH1J\nhqnFp/U6gC5btmq01yF0zTC1FYavL4/2VX5PY2YGdgB/0da7Ozmb/cvA+sagPnFp2ruB9R0sS5Ik\nzVP2YJ5S+npELAFuofh7/QfAypTSG2UHJ0mS5tZRBbiU0lpgbcmxSJKkDng2uyRJNddJOddzgN+j\nKB6zFPi1lNIDs74ppbZL/Iws+GJWPOOHPpc1f2TN3UE843nx5MqtlPSFBXmVktYcyq2albdGc+JZ\nMz7/Cl69lN3XRm7Pmn98/MbMT6i7vDV660j7fe2m3L4WebHkbqdytzuHKl5+1du1zNWZvZ3K3w7+\nT7XLb7O/7dh6gHXL21tmJ3vmR1IcJ7+e/Cp/kiSpZJ2cAPcQRT3+icvSJElSD3nMXJKkmnMwlySp\n5hzMJUmquY6uM8/1yOpHWXjM1Nrso5efyrIhq3ssSVIzYxvGGNu4fcq0fW/vb/v9XRnML7zzgqGv\nzS5JUiujq0Zn1IrfsXUn65bf29b7O7nO/EjgFP7/os/3R8TpwFsppVdzlydJkuankz3z5cDjFNeY\nJ+COxvSvAleXFJckSWpTJ9eZ/zOeOCdJUt9wUJYkqeay9swj4kbg14GfB/YCTwI3pJSeLy2ifisQ\nmxtPvxXFq/H6zF6VmYXrq65/vSBz+f2Wq+rvA5Bb//qIrPn7an2OZNZyP5hZa/2wau9pkV/Lvdr7\nBkTmbuiag5m13A+r9p4WX2j7vgHtD9G5e+bnAHcDHwVWAIcDD0fEuzKXI0mSSpK1Z55Sunjy84i4\nEvhvijuoPVFeWJIkqV3zPWZ+DMWfWW+VEIskSepAx4N5445pdwFPpJS2zzW/JEmqxnwqwK0FTgU+\nVlIskiSpAx0N5hFxD3AxcE5Kacdc81ubXZKk2WxrPCbb1/a7Oynneg9wKXBeSumVdt5jbXZJkmaz\nrPGYbAfwl229O/c687XAKuASYE9EHNd4aXdKqf2fEJIkqTS5J8BdCxwN/BPw2qTHZeWGJUmS2pV7\nnbnlXyVJ6jMOzpIk1VzuMfNrgeuAkxqTxoBbUkoPlRxXTlBZs+fW186tSVx3ufXQ8+tr59U8rlJk\n1u7OrX9ddS33kZHbs+avul42Ka/z5Ne/rq6W+61t18ou3JQZy8hItduR3NsYVF3LPVf2fQAy85Wv\n4r483l4t9x1bD7JueXvLzN0zfxW4ATiDooTrY8ADEeE1ZpIk9UjuMfN/nDbppoi4juLGK1aBkySp\nBzquABcRIxRnsS8E/qW0iCRJUpZOisYsA74LLALeAS5LKf2o7MAkSVJ7Ojmb/TngdOAs4B5gY0R8\nuNSoJElS27L3zFNKB4EXG0+/HxFnUZzhfk2r91ibXZKk1sY2jDG2ceqpZ/ve3t/2++dz17QJI8CC\n2WawNrskSa2NrhpldNXolGk7tu5k3fJ723p/7nXmtwEPAq8A7wGuAM4Fbs1ZjiRJKk/unvlPA18F\nlgK7gX8HVqaUHi87MEmS1J7c68w/XVUgkiSpM9ZmlySp5uZ1AlxEfA64DbgrpfQ75YRUsZG8mrvj\n49XWJK67yPw52G5N4m7ILAedX5966O4bkLdG1xzK6wtV3jcg+54BmcHk9p3c3B6qejuVu/IrXnxu\n38nN75rxvNr7udptb8566XjPPCI+QnE52jOdLkOSJM1fR4N5RBwF3Ad8Gni71IgkSVKWTvfM/wz4\nVkrpsTKDkSRJ+TqpzX458AtAm3dZlSRJVcotGnMCcBewIqXUP2cySZI0xHL3zM8E3gtsjfi/8+wW\nAOdGxGeBhSmlGae0WptdkqTWtm0YY2zD1Nrs+3dXV5t9M3DatGnrgWeBLzYbyMHa7JIkzWbZqlGW\nNanN/tdnVlCbPaW0B5jy0yEi9gBvppSezVmWJEkqRxkV4HJrb0iSpBLN+xaoKaVPlBGIJEnqTLQ4\nzF3OwiPOALZ8astV7R8zz42n4jKD2TLjzy3bOD5+Y9b8dV+dlar7yhnP7GuH2dda6bvY+y2gPtuu\n5eq31dmuScfMz0wpbZ1t3qy/2SPi5ogYn/bYPvc7JUlSVTr5m30bcAEw8dvlYHnhSJKkXJ0M5gdT\nSm+UHokkSepIJ2ezfyAi/isiXoiI+yLifaVHJUmS2pY7mD8FXAmsBK4FTga+ExFHlhyXJElqU27R\nmE2Tnm6LiKeBHwOXAS3L1DyyejMLF08t57r4xKO56O6VOR9fa9s2bB+q8rXbNozNqGY0qIaprTBc\n7R3bMMbokLQVhiu30F/57XY51ylSSrsj4nnglNnmu/DOFTMuTfvbS/5uPh9dO2Mbh2swH9uwfWg2\nCsPUVhiu9o5t3N43G/tuGKbcQn/ld77lXOdVAS4ijqIYyHfMZzmSJKlzudeZfykizo2IEyPil4Bv\nAgeADZVEJ0mS5pT7N/sJwP3AscAbwBPAL6aU3iw7MEmS1J7cE+BWZS5/EcCuZ3fNeGH/7v3s2Lqz\n2YfkfUK/1N2b0CL+/W83b+9IZnPHm62z/HBaKmt1tsxvPylp5fSsrbnlXEvqa63aW+evbqvY97X4\n3g5qOdeytstVb9dy9V1+2zRp7Fw017xV12b/TeBrlX2AJEmD74qU0v2zzVD1YH4sxTXpLwP7Kvsg\nSZIGzyLgJGDTXIezKx3MJUlS9eZ1aZokSeo9B3NJkmrOwVySpJpzMJckqeZ6MphHxGci4qWI2BsR\nT0XER3oRR9Ui4uaIGJ/22D73O/tfRJwTEQ80boc7HhGXNJnnloh4LSLeiYhHImLWGv79bK72RsS9\nTXL97V7FOx8RcWNEPB0RP4mI1yPimxHxwSbzDUR+22nvoOQ3Iq6NiGciYnfj8WREXDRtnoHIK8zd\n3kHJK/RgMI+ITwJ3ADcDHwaeATZFxJJux9Il24DjgOMbj4/3NpzSHAn8ALgemHFJRETcAHwWuAY4\nC9hDkecjuhlkiWZtb8ODTM11bpGlfnEOcDfwUWAFcDjwcES8a2KGAcvvnO1tGIT8vgrcAJwBnAk8\nBjwQEafCwOUV5mhvwyDkFVJKXX1Q3BP9TyY9D+A/gd/vdixdaOvNwNZex9GFdo4Dl0yb9hqwetLz\no4G9wGW9jrei9t4LfKPXsVXU3iWNNn98SPLbrL2DnN83gasGPa8t2jswee3qnnlEHE7x6+jRiWmp\nWKObgbO7GUsXfaDx1+wLEXFfRLyv1wFVLSJOpviFOznPPwH+lcHNM8D5jb9pn4uItRHxU70OqCTH\nUPwb8RYMRX6ntHeSgcpvRIxExOXAQuA7g57X6e2d9NJA5HVe9zPvwBJgAfD6tOmvAz/X5Vi64Sng\nSuA/gKXA5ym+NMtSSnt6GFfVjqfYGDbL8/EzZx8IDwJ/D7wE/CxwO/DtiDi78YO1liIigLuAJ1JK\nE+d7DGx+W7QXBii/EbEM+C5FdbF3KPa6X4iIsxnAvLZqb+PlgclrtwfzoZJS2jTp6baIeBr4MXAZ\nxd87GhAppa9PejoWET8EXgDOBx7vSVDlWAucCnys14F0SdP2Dlh+nwNOBxYDvwFsjIjzehtSpZq2\nN6X0/UHKa7dPgNsFHKI42WCy44A+v8XW/KWUdgPPA7U9O7RNOynOhRjKPAOklF6i6O+1zXVE3ANc\nDJyfUtox6aWBzO8s7Z2hzvlNKR1MKb3YGMzWUPyNfh0DmtdZ2tts3trmtauDeUrpALAFuGBiWuNv\nrQuAJ7sZSy9ExFEUnWTWDUXdNb4QO5ma56MpzhYe+DwDRMQJwLHUNNeNge1S4JdTSq9Mfm0Q8ztb\ne1vMX+v8TjMCLBjEvLYwQnG4d4Y657UXf7N/GVgfEVuAp4HVwLuB9T2IpVIR8SXgWxR/rf8M8AfA\nAWBDL+MqQ0QcSfHDZOLOv++PiNOBt1JKr1Icd7wpIn5Ecde8P6S4auEfehDuvM3W3sbjZopjbzsb\n8/0Rxb8wm2Yurb9FxFqKy3MuAfZExMSe2u6U0sTdDwcmv3O1t5H7gchvRNxGcZz4FeA9wBXAucCt\njVkGJq8we3sHKa9A9y9Na5xTcD1FR9lLcWLC8l6f1l9ROzdQfBH2UnSm+4GTex1XSW07j+LynUPT\nHusmzfN5iktd3qH4cpzS67iraC/FiTUPUWwQ9gEvAl8B3tvruDtsa7N2HgJ+a9p8A5Hfudo7SPkF\n/qoR/95Gex4GPjGIeZ2rvYOU15SSt0CVJKnurM0uSVLNOZhLklRzDuaSJNWcg7kkSTXnYC5JUs05\nmEuSVHMO5pIk1ZyDuSRJNedgLklSzTmYS5JUcw7mkiTVnIO5JEk1978M4TJVqhYNwgAAAABJRU5E\nrkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n", + "ready to train..." + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(NNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "for i, (grad, var) in enumerate(gradients):\n", + " if grad is not None:\n", + " gradients[i] = (tf.clip_by_value(grad, -10, 10), var)\n", + " \n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model overview...\n", + "\tvariable \"W1:0\" has 2048 parameters\n", + "\tvariable \"b1:0\" has 128 parameters\n", + "\tvariable \"W2:0\" has 32768 parameters\n", + "\tvariable \"b2:0\" has 256 parameters\n", + "\tvariable \"W_z:0\" has 12288 parameters\n", + "\tvariable \"W_v:0\" has 1536 parameters\n", + "\tvariable \"W_r:0\" has 60 parameters\n", + "total of 49084 parameters\n" + ] + } + ], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loaded model: models/model.ckpt-7000\n" + ] + } + ], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 7053/10000" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtsteps\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtsteps\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mstep_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mloss_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mglobal_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + " global_step = i\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=global_step)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA94AAAHHCAYAAABJK4BRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XmcHFW99/HPL5MwgUASIECisikgkgAyiQubRAFR76Oo\neDEjKqCCuFy8uY8iCCSQK8sVJYreXBGF4AMkFy+IQWWLwAVlkwwiAaKybwkQQhLISmZ+zx+nJunp\n6Z6u093V23zfr1e/kqk+depXdarr1KnlHHN3RERERERERCQbQ+odgIiIiIiIiEgrU8NbRERERERE\nJENqeIuIiIiIiIhkSA1vERERERERkQyp4S0iIiIiIiKSITW8RURERERERDKkhreIiIiIiIhIhtTw\nFhEREREREcmQGt4iIiIiIiIiGVLDW0RERERERCRDaniLiIiIiIiIZEgNb6kJMzvOzHrMbKd6x1IJ\nMzvFzB6pQj5fNrOnzWxYNeKqFjM7y8x66h1HoyinvBu1bEVEpP4K1bPVPkdqhHOugepPMxuaH5vq\nThkM1PCWWvHk0xDMbH8zm25mIyPm2Qo4BTi/CiHMBjYDvlyFvKopVTmVs/3qqcblPZvGLFsREam/\nQvVs9DlSiXqtrudcA9WfZmbAz4BLzezNOV/NRnWntDg1vKVWfgls7u7P1DuQxAHANGB0xDxfBNqA\nuflfmNkxZnZXcoV5vZnNzvnuv5Lpy8zsOjPbzt3XAZcD/1bZatRNOduvnqpa3gNpgbIVEZHaKucc\naaB6rd7nXAPVn/9BiO9o4Dwz2xpUd8rgoIa31IQH6+sdRw4rY57jgHmF1sPdr3T3A4AHgUfc/bic\nry8C5gE7u/vH3f3lZPrVwC5mNrmMWOqtnO1XOlOzLbLIlyqXdwrNXLYiIpInw/qp3HOkovVaA5xz\nHUfx+vOn7n67uy8DvkZooPdS3SktTQ1vqYn8941633Eys7eZ2Wwze9XMlpvZpWY2PG/e3rRvN7Or\nzWyFmS01sx+aWXtOutlm9mSBZfd5n8rMpgPfS/58Ksm7e6B3ocxsF2AfYH6JVb0E2NvMJibzbQec\nDBzl7q/lJnT3LmAZcGSJPDGzncxslpktMrPVyfpfbWY7F1rXlNv1IDP7s5mtMbN/mNmJpeJI5htw\n+5UR6zvM7CozWwbcmfP9ZDO7Pze+Iu/GvSlZvyVmttbMFprZ8WnjLbKOu5CuvAuKKVsREamNiPOJ\nUvXTgPVOTrpU9Wz+OVLOMn5hZs8ny3giqVuHpqiHC77jbWb7mdkNyXq/Zmbzzew9RbZRyfOIIuuy\nCwPUn+7+RM7/X3P3pTl/q+6Ulja03gHIoJH/vlHv/68GngBOBTqALwEvAqcVSftkkva9hAbtaMKV\n1ULLKLbsa4A9gCnAN4BXkukvU9wBSR5dA6QBuBK4APiSmT0KnAtMdffuIum7gANL5AnwLsI6zwGe\nA3YBvgrcZmZ7ufvaJF2q7WpmE4CbgJcIj6oNA85K/i6l1PaLjfVXwN+T2CyJbz/gBuAF4EzCsepM\nYGnOfJjZ9sC9QDfhyYKlwIeBX5jZVu5+EXBtiXgLSVveA0lbtiIiUhsx5xNQuH5KU+/E1rN9zlPM\nbBzwZ2AkcDHwN+DNwKeALShdr/U7HzKzvYA7gBWEd683EN6nvt3M3ufufy6wjUqdnxVSaf2pulNa\nl7vro0/mH+BYQiW1U/L3dKAH+FleumuAl/Km9aa9Nm/6T5I8JyR/XwY8UWDZ04HuvGn/NzeeFPHP\nSNJvkSLtbELFdjEwtkTanwKvp8izvcC0dyfb5ZgC22rA7Qr8GlgFvDln2tuBN/K3VZF4im6/MmL9\nfwXSzwNeA3bImfZWYH1ufMDPCY370XnzX0W4at5e7fIGPku4uPLDnGmbEU7OxsWWrT766KOPPrX5\nRJxPDFQ/pa13Utez9D9HujxJt98A6zJQPXxs/ndJPGsIr731ThubnK/cVmAblTw/KxJXwfpTdac+\n+rgeNZe6ckLjNNedwLZmtmWBtP+ZN+3HhCvQH8kmvD62BTa4++oUaS8BtgIWuvuSEmlfBTYv9fiW\nh05HgI3DcGxDuBK9nHAluk9yBtiuZjYE+CDwa3d/PmcZfyNcna9IpbEm8R0KXOfuL+bk+wThLniu\nTwLXA21mtm3vB7gZGFVgeWkVLG8zezuh8X8t8IWcryYC2wO55Z2qbEVEpKbSnk8Uqkth4HpnNNBR\nST1rZkZ41Hqeuz8QtWbF8xwCHJ7E83ROPEsIFwwOyjvvijk/y9ev/lTdKRKo4S31lt/j5qvJv1sX\nSPtY3t+PE67K7lLlmCo1kXA1vDNF2t7OUQYc9sPMhpvZDDN7BlhHeLTtJULjclSBWQbartsBm9N/\ne0J4nK0iZcSa/17+9gPEt3GahffnRwMnEh6vy/1cmpNXNR0IXAdMBu7PmX4AcJe755ZjqrIVEZGa\nS3s+0ad+SlHvOKHeqaSe3Y7wiPnDJdLF2I7wiPrfC3z3KKE9sGPe9Jjzs1JUd4qgd7yl/oq9+5ym\nF+pC42AW0lZkeoxXgKFmNsLdVxVLZGZHE67cfh+YaWbj3X2gynNrYHXuXeIifkJ4dGwmcA/h0TAH\n/pvCF9Aq2a6Vio11TZnL6c3rCsJjeYX8tcy8C5a3u18KYGYfJ7za0OsAwrtzudKWrYiI1Fex84f8\n+iltvVON8456K/c8ol/9qbpTJFDDW5rJ7sDTOX/vRqgEe69Iv0rh8Sx3KTAt9krqouTfXYGFhRKY\n2SGEd7ovMrPRhM5LTiR0fFLMroSrzaUcBcx291NyltdOeeNov0w4mdi9wHd7psxjoO1XaawvAWsJ\n5ZsvN+aXCe+Bt7n7rSXyrFp5WxhzdCKhk5te+wM/yMsjbdmKiEhtlTqfKCZVvZM82l1uPfsysBKY\nUCJdTL32MrCa8I55vncQ7vY/G5HfQArWn6o7RfSouTQPI4z3mOtkQsVzY/L348CopCfRMFPoGfTj\nBfLrvYuZtjF4dxLDpILBhd5CD/WkN1N3X07oiOSzuUOUFNAB3JVi+d30/72eTBlX1d29h/CO2cfN\n7C29083sHYR30tIYaPtVFGsS3/wkvrE58e0GfCgv3TXAUWY2Pj8fMxuTMt5CBirvtwHL3f3ZZDlv\nJTxC/+e8dGnLVkREameg84n8fkT6SFvvVFLPJo9dXwd81MwG6qckdb2WxHMzcGTekGU7EF6Lu9Pd\nXy+VT0rF6k/VnTLo6Y63NJNdzew3hIb2AcAxwBXu/lDy/VzgP4DrzOwiYARwEuF9qvzKawGhYjjX\nzOYSeg+d5+4FH3t29yfNbCFwGKHX8o3M7N3Aae7+ibzZfp7E+Gngl/l5WhjrextCBVvKb4HPmdlK\n4BHCVeJDCe9Pl2M6oRH7RzObRRjm5OuEq9P7pJh/oO1XjVjPIpyc3GVm/0U4Vn0tiW/fnHSnEt4Z\nu9fMLkmWtw3hqvoHgN7Gd9XKm3C3Y5iZWXKC9BXgPnd/ozdBZNmKiEhtFTufKPhEW5609U4l9ex3\nCJ2h3WFmPyPcAX4T4W7xge6+ksh6DTiDUKf9KYmnm/BU3mbAKUXmiTZA/am6UwY93fGWZuGEBuw6\n4DzCmJkXEcaVDAnclxHubq8iNMA/R6ggf9svM/f7CZXQPoT3ja4idD4ykEsJV6DbIbyrZGZ3AH8C\n3mdmG69im9l7Cb2kOnCRmV2T9O6d65+Bp9399hTrfzKh8f4ZwvvjOxAqtdcpowOS5GLFBwmPdZ9N\nGLt0GikruxLb7xuVxuruXYQTlmWEoUm+QGiM/4HwGHpvupcIQ5VdCnyCsM17x2M9JSddxeWdk9ff\nCPvehWZ2KqGH29vy5o0pWxERqZ2S5xMDzpy+3im7nnX3F4D3EMYR/wzwI8JwXLcSHhmPrtfc/RHg\nYOAhwrnRmYRH6ycneVVTv/pTdacIWN+OBEUaj5lNJ1RW2yWN63rFMZLwOPsp7n5ZqfQl8toMeAo4\n191/UoXwBgUz+zWwl7sXek+t2svqV95mNgy4EPiRuz+WvMrwD2Bvd38ySaOyFRFpQI1yPtHq8utP\n1Z0ige54i6SUPNp1AfCtKmR3PGFMy0JjhAphWLK8v3cnjLGaf4U8E0XK+53A5wnDskAYs/303hOH\nhMpWREQGrQL1p+pOEXTHW5qArlAPTmb2AuH9sCcIPdOfRHhHrsPdH69TTEMJj+itB8YB97j7f9cj\nFhERiaPzifpQ3SkSqHM1EWlUNwBTgLGEd/HuAr5Tr0Y3gLtvAL5br+WLiIg0G9WdIoHueIuIiIiI\niIhkKNM73ma2LXAEobOEtQOnFhERaUnDCa9L3OTur9Q5lqpRHS8iIpK+js/6UfMjgCszXoaIiEgz\nOIYw5E+rUB0vIiISlKzjy2p4m9nXgG8S3r18EPgXd/9zgaRPARx5xUcZs+e2fb64ZeofOHzmof1m\nGDJpdlQsPfcflzptW2TeMYo9sH8T4cwkX8+C49PnHfk2wKWT0hfrF+7fEJn3sCLf3EgYdrkScSv6\nxfvfSJ22WmVfaC1jX9aIKfssFduvbpk6n8NnHtZvevGyr0o0Uam/uCBuvy2k2HpGifxxxhzfoveT\nIrFU41ib5T5bveNb/6Nt7H4SE0vc72EpcC0kdWKji63jr/gm7Llj3y+mXgIzT+g/w9p3xcUypDt9\n2s3+Hpf3+j3Sp20rEsc3vwPfP7dA+mfT5/3KO0akTwxs+0+r0uf9u7i8x3yhcN5Tn4SZu/adFts1\nWffyuPRtM9Knfep9Y6Py3nnlkoLT/+10uPCcvtNWj2iLyrutO/1Ou3nkNvSV6dO+skfhsp8+dS1n\nzxzeb/qYj6TfrwC8J33aVSuismbLqyMSFzkUT50BM6cV+OKe9Fl73G5F94T0aYe+Gpc3mxWePPVs\nmDm9wBcR9ZkvjQtl/dvSp93sxbi8u48rPP2ba+D7m/edNjR20LmYantW+qSPvg6ffRBIUcdHN7zN\n7NPAD4ATgfuAqcBNZraHe7+iWwswZs9tGdfRd+9tH93ebxrEj2/WUyCPYuIOnXGK7d/DCd035ouJ\nO/41/PR71riO9I3XoMgvv+iaxohb0XEd61OnrVbZF1rL6IZ3RNlnqdh+NbzIb7N42VclmqjU8ftt\nf+2jiq1nhNiGd0Ta6P2kSCzVONZmuc9W7/jW/9cZu5/ExVLW76HhH8cup47fc0fo2K3vF6O36D8N\nYPV+FhVP24b0hdIelzXr9o2Jo/D0kSNhvwL5DI1o777YEVdD7RCx68XmPXbLwtNHDYWOvO98dVTW\nbIisiIdGnNyP6Ij7Pe5RpNEzahR05JXn6yNj99n06Ue8FHcAjLnY8eK+hTf4yNHGPgX2i7GRhzSP\nuCj2WuTJ/cjxEYnbC08eNRI69i7wxeL0WftOEXEAGyKOKcNejsub/tdKgAHWM6bhHbFNANZFlE97\nxEVIKH6cGGmwX953w/aMyzuq2h4VmXdQso4vZxzvqcDF7v5Ld19EGOJnNfCFMvISERGRxqE6XkRE\nJANRDW8zGwZMBP7QO81Dt+jzgf2rG5qIiIjUiup4ERGR7MTe8R5DeGo3/4n9FwnvgomIiEhzUh0v\nIiKSkax7NQdC5z7to/u+hDFqp5G1WHTdRfSz0OQGx5oOhrUcP2WveodQE+M7B8l6DpLybJxf50PA\nwrxpDf9qd0WmXhLe6c610/b1iaXWPn1UvSOojc4x9Y6gNqZ8st4RZO/jU7LsKLVxdH6s3hHURueR\n9Y6gNj7dILvtnBfCJ9eKiO5kYhveS4FuYIe86TsAhbuIBA6feWjlnRg1qUY5FcxeoZ4dWs9gWMvx\nnTG9mjSvCYNmPdXwrq296X+kWAz8rA6xRCurjp95QuGO1AaDKZ+qdwS10bldvSOojc5BcCHlE50N\n0oLJ2GBpkA6W9ZySZf++ETrfFD65ulbAxD+lmz/qUXN3fwNYAGwcm8bMLPn7rpi8REREpHGojhcR\nEclOOY+aXwjMNrMFbBpqZAtgdhXjEhERkdpTHS8iIpKB6Ia3u19tZmOAGYTHz/4CHOHusSPSiYiI\nSANRHS8iIpKNsjpXc/dZwKwqxyIiIiJ1pjpeRESk+qIb3mZ2MPAtwlif44CPu/u8agcWEVDqpN09\np2YYSHOK2HwAnN6zPptAmljkJmSInZdZ5j09p8XN0DDiVvS7FtE5TOQ2PKMnonvKLLlHJR/Sdn5U\n+p5BcDyM3ISc05Z+v4o5Fi7ueoNLJ8XFUi9l1fFtpD6bWN4+OiqecateTZ94RFTWvDYi/QxjXlgV\nl3mEyN2UGc+mTzvtbyvjMr86fVL7R1zWQ2M3YcSxex1xPS+tay+dptdrbVtF5b3NuvT77Mo3xZ2G\nb7VsQ+q0Q+iOynvBc1HJeUfE7219+rABuH2P9GknvzUub86MSDsqLuvXRqXfsbZZuS4u89jOxSK3\neYwXRuT3v1ncDru8FJX32tfTHxG3iTxd+9/906edfHlExk8BWXSulhhBePTsq8TXFyIiItK4VMeL\niIhkoJx3vG8EboSNvZ2KiIhIC1AdLyIiko1y7niLiIiIiIiISEpqeIuIiIiIiIhkqKxezWPdMvUP\ntI/u2+HA+Cl7MaFzr1osXkREpCYenvMwD899pM+0tcsjO9FpMlN/CqO37DttymTofH9dwhEREcnE\nnLthzj19p61YnX7+mjS8D595KOM6xtZiUSIiInUzvnM84zvH95m2uGsJl066rE4RZW/mSdCxe72j\nEBERyVbn/uGTq+spmDgt3fx61FxEREREREQkQ+WM4z0C2I1NIyy+1cz2BZa5e8TokiIiItJIVMeL\niIhko5xHzScBtxHG93TgB8n0y4EvVCkuERERqT3V8SIiIhkoZxzv/0WPqIuIiLQc1fEiIiLZqEnn\napiFTwo93adGZT1kyHmp0/b0nBaVN+7p42g7PyrrmFhSbrqNTu9+I3Xa7w4ZFpX3GT3p847YfACc\n0xYZS/f6uAU0odjfg/QX83toKJE//J4NkcfOiGNW9LGzQUQfOzek31fOiTp21qaqrZtuYEO6pFt2\nvx6XdVv6tENXRWXNlqvSzxC5K+ERRT569cqovM98OX3ap7aPyppdIvJmp7i8H9stLv1u16VPO5Tu\nqLzbIwYa2GyLuHONnrb0e8tWr6T84ZRheOQ5UsebIhcwvnSSXptHZv22W9KnXbko7vg68vKIbb42\nKmva10Wcb8QOdhFbjUT8JGKPb7u+/GLqtK9u1146UY5tJqXfMBb3s+eQmyMSz4pIuzx90qir2mZ2\nmpndZ2YrzexFM/u1me0Rk4eIiIg0HtXxIiIi2Yl9nOxg4MfAe4DDgGHAzWYWezFLREREGovqeBER\nkYxEPbjg7h/J/dvMjgNeAiYCf6xeWCIiIlJLquNFRESyU2kHKqMJvZ4uq0IsIiIi0jhUx4uIiFRJ\n2Q1vMzPgh8Af3f2R6oUkIiIi9aQ6XkREpLoq6Wp1FrAXcGCphLdMnU/7qL692o3v3IsJnRFdIoqI\niDS8hcknV2TXuI0hdR0/9RIYvUXfaVMOgc7JmcQlIiJSF3OehTnP9Z22IqIz+7Ia3mb2E+AjwMHu\nvrhU+sNnHsa4jrHlLEpERKSJTEg+uRYDl9QhlvLE1vEzT4COyKGiREREmk3njuGTq2s5TLwt3fzR\nDe+kQj4SOMTdn4mdX0RERBqT6ngREZFsRDW8zWwW0Al8DFhlZjskX61w96Z8lk5ERERUx4uIiGQp\ntnO1k4CRwO3ACzmfo6sbloiIiNSY6ngREZGMxI7jXenwYyIiItKAVMeLiIhkJ/ZR85OArwC7JJMe\nBma4+41Vi8gi03tM2pjEMKTt/NRpe7pPjco7UzHbMG6TRG3Cc9qGReV9endEt4CRIlczqjxj9pPM\n8+45LSp9nLitGFOeWe4rsXmf0b0+ddohQ+LKJ+anOWTIeVF5Rx+DYn8UMVnHHCeGxJVPzFb8bmTe\nccegmNKMrdjqo+w6vo3UZxPr2zaLi2nL9DvTVkM3ROW9ZkR76USJoRvWReU9bHX6tMNXRWUNr6dP\n+uaRkXlHxLJhYlzWu90al56IbbiO9GUJsKEtfdr1xO2z7UPT7yvrRsTts+0j0qfdMqarZQi/4wjr\nrkmfdvi+cXm/ZVFE4j3jtiFnRqQdFZd199CIa5cje+IyjxVRnh5b9luUTtNr+Mi4YydvS5/Uj4rL\n2mZFJH4pIm3EMTn26vazwLeBDmAicCswz8z2isxHREREGovqeBERkYzEPmr+u7xJZ5jZV4D3AI9U\nLSoRERGpKdXxIiIi2SlrHG8AMxtC6HClHbizahGJiIhIXamOFxERqa5yxvGeANwNDCe8gXO0uz9W\n7cBERESktlTHi4iIZKOcHkwXAfsC7wZ+Asw1s/2qGpWIiIjUg+p4ERGRDETf8Xb3DcATyZ8PmNm7\nCb2gnlhsnlumzqd9VN8eJ8d37sWEzvGxixcREWlgDwEL86atrUcgZSmnjp/6Uxi9Zd9pUyZD5/sz\nClJERKQO5rwcPrlWdKefv+x3vHMMoUSn9YfPPIxxHWOrsCgREZFGtnfyybUY+FkdYqmKknX8zJOg\nY/caRSMiIlInnduFT66u12Hig+nmjx3H+1zgBuAZYCvgGOB9wHdj8hEREZHGojpeREQkO7F3vLcH\nLgfGASuAvwJHuPtt1Q5MREREakp1vIiISEZix/H+UlaBiIiISP2ojhcREclONd7xLs09fFIYMuT8\njINJz9KFDMCQtri4e3pOS5025abb6Jwhw+JmyIpbVPJz2uLiPqN7fVT6KEPSx97TfWpc1kPT7yux\neTcSixgz4fTuN6LyPmdo+n0lNm8soux7Issn4rfcFnlMiT8GZbdvRWxCTu+JLJ+IbXhO22ZRWccc\ng07vSX/8Wdz1BpdOigqluXQDG9IlXW9xZdLWljJjgPaItMB62ksn6k07PK6+GbYiYkeNrONX7pM+\n7QPr4vI+5Jn0af+xPC7vZR+IS3/gdenTrmbzqLzXjEj/W19tW0TlvVlb+n1ls7Q/nIStiEgclzVv\nrIxLv37z9E2IIQvignls2/Rp97orKuv+fV8OJHIbvta2Veq0W3W/GpW3pz9cAdA9YM8cfQ2L6BwM\noP3hiMQXx+XNO9MnXRZxLAQYdmz6tCNnRWT8NJDyHe9yhhPbyMxONbMeM7uwknxERESksaiOFxER\nqZ6yG95m9i7C8CIp2/giIiLSDFTHi4iIVFdZDW8z2xK4AvgSEPmwkYiIiDQq1fEiIiLVV+4d7/8E\nrnf3W6sZjIiIiNSd6ngREZEqi+5czcymEF59b+WuYkRERAYd1fEiIiLZiGp4m9lbgB8Ch7l76q5o\nb5n6B9pH9+2Ob/yUvZjQuVfM4kVERBraw3Me5uG5j/SZtnZ5ZNfSdVJuHT/1Ehid1/HzlEOgc3JV\nwxMREamrOffAnPv6TluxOv38sXe8JwLbAV1mGweMaQPeZ2ZfB9rd+w9+dfjMQxnXMTZyUSIiIs1l\nfOd4xneO7zNtcdcSLp10WZ0iilJWHT/zBOjYrYZRioiI1EHne8MnV9fTMPHsdPPHNrznA3vnTZsN\nPAqcX6hCFhERkaagOl5ERCQjUQ1vd18F9HmGzsxWAa+4+6PVDExERERqR3W8iIhIdsoexzuHroCL\niIi0JtXxIiIiVRDdq3k+d/9ANQIRERGRxqI6XkREpDos5pUtM5sOTM+bvMjdC3ZPbmYdwIIvLjg+\nfedqWb5CtrGvmJRiYokMe8jQ81On7ek5LSrvRtmEWb8NGFWcscHE7isxMtwwQ9oaZ79qlE2YZRyZ\naqDXaWP2K4jft7LSKMfCxV1L+MXEywAmuntXVjFVqtw6/pb7N2efjrZUy3jadq4syAHs7E9HpX/Q\n9k2d9q3+eFTet/v7U6f90tCTo/Lm0+9On/Z7a+Py3jl997z2jW2isp5/4YFR6SewMHXa++xdcXl7\n+ryX2LiovLdhWeq0yzxuG/7D0/di+Pmh+d01lPCn/J9+CZ+OSHtMXNZ8MaJt8mJc1s8fkH6bP03c\n8WqDpTsOAqz39tKJcmzL0qj0TvpK6j5/T1TeX277ZvrEsQV09+Hp06bf3CGU0T2p0576trNSp13S\ntZjLJv0CUtTx5dzxXggcChtLdEMZeYiIiEjjUR0vIiKSgXIa3hvc/eWqRyIiIiL1pjpeREQkA+V0\nrra7mT1vZo+b2RVmtmPVoxIREZF6UB0vIiKSgdiG9z3AccARwEnArsAdZjaiynGJiIhIbamOFxER\nyUhUw9vdb3L3a9x9obvfAnwE2Bo4OnbBC+c8HDtLU1o455HSiVrA4CnP1l/PwbCOMJjWU8cgSaea\ndfy1c96oenyN6Po5a+odQm34tfWOoCbmzYnsjK4Z+UP1jqAm/JY59Q6hJm6Y81q9Q6iNm1ujPCsa\nTszdV5jZ34EBu1m8Zep82kf17b3vtedfY0Ln+EoW3xQenvsIEzoLdgjbUh6e88jgKM9BsJ6DYR1h\nEK2njkE1tXDOwzycd7Fj3Yp1dYqmMmnr+DOnrmPk6L496C5+vodPdg7LMryGcP3ctXy0c/N6h1ED\n1wKfrHcQmZs3dx0f6xxe7zAythCI7PG8Gc2fC4d31juKzN049zU+3LlVvcPI3s1z4YP1L8+H5yzk\nkbl9L/SvW56+jq+o4W1mWxIq5F8OlO7wmYf1G07svz/2q0oWLSIi0nAmdI7vdwEgZzixppK2jv/3\nme39hhP73JGD5E6wiIgMGuM7JzC+c0KfaTnDiZUU9ai5mV1gZu8zs53N7ADg18AbQGvc/xcRERmk\nVMeLiIhkJ/aO91uAq4BtgZeBPwLvdfdXqh2YiIiI1JTqeBERkYxENbzdPfbh+uEASx9d2u+LdSvW\nsbhrSaGFRC4iglnpNLliYimStNh6DonIuqfQdhoolDpswkLrmWUcA8VSUGwwRTIvut/GyHDDVGO/\nKraOVdqEVRETS8w+21CqtJ+sW175esbsVxB/zKqGWh+DYvbvnDqwoV8eLbeO/8einn5frFzu/LWr\nu9/0xZbdI+grvP/yBvIPez112tVeuLO415b3sLCr/3dP+7L0gfjf0qcFWBZx6vZQZP8CXqxzsZXg\nD/ZN+tKyst2LAAAgAElEQVSoqKz/3rUqKn0P6cvzMYvsYCqiPF+x1VFZj2R96rQrPS7vZ/zV9Il9\ncZEv1hb+7m9dUbFErCbEVgkPpz94F70s+PoKvMA6/XV4+v1qCXHls8HaSidKvOEbovIeTeHf5mvL\ne3i0q/93TvpK6pnoa6uPpk8am/eibQtPf305LMorz/SbO4SyVf+6qpglK4r9fvpbuih9HW+e4ZmJ\nmX0GuDKzBYiIiDSPY9z9qnoHUS2q40VERDYqWcdn3fDeljAe6FNQ5FKNiIhIaxsO7ALc1EqPbauO\nFxERSV/HZ9rwFhERERERERnsono1FxEREREREZE4aniLiIiIiIiIZEgNbxEREREREZEMqeEtIiIi\nIiIikiE1vEVEREREREQyVJeGt5l9zcyeNLM1ZnaPmb2rHnFkxcymm1lP3ueResdVCTM72Mzmmdnz\nyfp8rECaGWb2gpmtNrNbzGy3esRaiVLraWaXFSjb39cr3nKY2Wlmdp+ZrTSzF83s12a2R4F0TV2e\nadazRcrzJDN70MxWJJ+7zOxDeWmauiyh9Hq2Qlm2CtXxzUd1/Mbvm/44ojq+T5pWKE/V8bRGWUId\nGt5m9mngB8B0YD/gQeAmMxtT61gythDYARibfA6qbzgVGwH8Bfgq0G8MOjP7NvB14ETg3cAqQrlu\nVssgq2DA9UzcQN+y7axNaFVzMPBj4D3AYcAw4GYz27w3QYuUZ8n1TDR7eT4LfBvoACYCtwLzzGwv\naJmyhBLrmWj2smx6quOblur4TZr9OKI6vq9mL0/V8Zs0e1mCu9f0A9wD/CjnbwOeA06pdSwZruN0\noKvecWS4fj3Ax/KmvQBMzfl7JLAGOLre8VZ5PS8Drq13bFVezzHJuh7U4uVZaD1brjyT9XoFOL5V\ny7LIerZkWTbbR3V8839Ux7fWcUR1fGuVZ7JequOb9FPTO95mNoxwFeMPvdM8bM35wP61jKUGdk8e\nZXrczK4wsx3rHVBWzGxXwpWn3HJdCdxL65UrwOTksaZFZjbLzLapd0AVGk248r8MWro8+6xnjpYp\nTzMbYmZTgHbgjlYty/z1zPmqZcqyGamOb02tehwZQKsdR1THt0h5qo5v/rIcWuPljQHagBfzpr8I\nvL3GsWTpHuA44G/AOOAswg9kgruvqmNcWRlLONgVKtextQ8nUzcA1wBPAm8DzgN+b2b7JyeYTcXM\nDPgh8Ed3731HseXKs8h6QouUp5lNAO4GhgOrCVe6Hzez/Wmhsiy2nsnXLVGWTU51vOr4ZtdSxxHV\n8a1RnqrjgRYpy1o3vAcFd78p58+FZnYf8DRwNOFRCWlS7n51zp8Pm9lDwOPAZOC2ugRVmVnAXsCB\n9Q4kYwXXs4XKcxGwLzAK+BQw18wOqW9ImSi4nu7+QAuVpTQ41fGtqwWPI6rjN2nm8lQd3yJlWevO\n1ZYC3YQX43PtACypcSw14+4rgL8DTdfLYEpLCO/xDapyBXD3Jwn7ddOVrZn9BPgIMNndF+d81VLl\nOcB69tOs5enuG9z9iaRyOp3wmNlXaLGyHGA9C6VtyrJscqrjW1NLHUdiNPNxRHV8f81anqrjC6Zt\nyrKsacPb3d8AFgCH9k5LHg85FLirlrHUkpltSdgxBjwgNKtk519C33IdSehpsmXLFcDM3gJsS5OV\nbVJRHQm8392fyf2ulcpzoPUskr4py7OAIUBbK5VlEUMIjzb300Jl2TRUx7fmvjYIjiNFNetxRHV8\n0fRNWZ4FqI5v0rKsx6PmFwKzzWwBcB8wFdgCmF2HWDJhZhcA1xMePXszcDbwBjCnnnFVwsxGEE4s\nLJn0VjPbF1jm7s8S3q05w8weA54C/p3Qk+1v6hBu2QZaz+QznfCOyZIk3X8Q7nTc1D+3xmRmswhD\nMHwMWGVmvVdKV7j72uT/TV+epdYzKetWKM9zCe8+PQNsBRwDvA/4bpKk6csSBl7PVinLFqE6vgmp\njlcdT5OVp+p41fE0WVkCtR9OLHn//auEnWMN4SX6SfWII8P1m0PY6dcQdqCrgF3rHVeF63QIYZiG\n7rzPpTlpziIMa7Ca8EPYrd5xV3M9CZ093Ej40a8FngD+C9iu3nFHrmOh9esGPp+XrqnLs9R6tlB5\n/jyJfU2yLjcDH2ilsiy1nq1Slq3yUR3ffB/V8arjm608Vce3TlmWWs9WKUt3x5IVEhEREREREZEM\n1LpzNREREREREZFBRQ1vERERERERkQyp4S0iIiIiIiKSITW8RURERERERDKkhreIiIiIiIhIhtTw\nFhEREREREcmQGt4iIiIiIiIiGVLDW0RERERERCRDaniLiIiIiIiIZEgNbxEREREREZEMqeEtIiIi\nIiIikiE1vEVEREREREQypIa3iIiIiIiISIbU8BYRERERERHJkBreIiIiIiIiIhlSw1tEREREREQk\nQ2p4i4iIiIiIiGRIDW8RERERERGRDKnhLSIiIiIiIpIhNbxFREREREREMqSGt4iIiIiIiEiG1PAW\nERERERERyZAa3iIiIiIiIiIZUsNbREREREREJENqeIuIiIiIiIhkSA1vERERERERkQyp4S0iIiIi\nIiKSITW8RURERERERDKkhreIiIiIiIhIhtTwFhEREREREcmQGt4iIiIiIiIiGVLDW0RERERERCRD\naniLiIiIiIiIZEgNbxEREREREZEMqeEtIiIiIiIikiE1vEVEREREREQypIa3iIiIiIiISIbU8BYR\nERERERHJkBreIiIiIiIiIhlSw1tEREREREQkQ2p4i4iIiIiIiGRIDW8RERERERGRDKnhLSIiIiIi\nIpIhNbxFREREREREMqSGt4iIiIiIiEiG1PCWQcPMjjOzHjPbqd6xVKLS9TCzs5L5t6lXDNVgZqeY\n2SMZL+PLZva0mQ3LcjkiIiIi0trU8JbBxJPPRma2v5lNN7ORdYqpHP3WI4v5S2ybSmOoiJltBZwC\nnJ/xomYDmwFfzng5IiIiItLC1PCWweSXwObu/kzOtAOAacDo+oRUlkLrkYWBtk2tYijmi0AbMDfL\nhbj7OuBy4N+yXI6IiIiItDY1vKXlmdkWAB6sz/+6DiFVpMh6ZKHotqlhDMUcB8yrUQxXA7uY2eQa\nLEtEREREWpAa3tKQzGzv5B3i/5MzrSOZdn9e2hvM7O7k/73vL7/DzK4ys2XAncl3fd5LNrPpwPeS\nbJ5KvuvO+f5NZnapmS0xs7VmttDMjs8q9rTLLPZ+tZlNNrP7zWyNmf3DzE7s3R5FQt3azGab2atm\ntjxZ7vCU2yZ/W/Zu97cVy7PCWHPn3QXYB5hfKm01uHsXsAw4shbLExEREZHWM7TeAYgUsRBYDrwP\n+G0y7WCgB9jXzLZ099fNzID9gZ8maXrfO/4V8HfgNDbduc1/L/kaYA9gCvAN4JVk+stmtj1wL9AN\nXAQsBT4M/MLMtnL3i6ode8QyC72rvh9wA/ACcCbht31mkkehd7GNcCf3CeBUoAM4AXgx2WbXFts2\nRWLo/X9+nl/KybPcWPMdkKTrSpG2WrqAA2u4PBERERFpIWp4S0NydzezPxEarL0OBn5NuPN4AHAz\n8E5gJMld7RwPuPvnSixjoZl1ERqXv8l9X9nMfkxonL7T3Zcnk39mZlcBZ5nZxcn7v9WM/dxylwmc\nDWwADnD3F5N1uBpYNMAmWODuJ+as8xjCu9OnuftDxbZNCUXzrDDWXHsm/z6ZO9HMPgvsCwxz939N\nph0ETHf3w81sc8JFkX9x99+nXFavJ4DPRs4jIiIiIgLoUXNpbHcCHUmDCeAg4PfAg2xq1PbeSf5T\nznwOXFzhsj8JXA+0mdm2vR9Cg3kU4W5utWL/YyXLNLMhwKHAdb0NWQB3f4JwZ7mQQtvoTmBbM9uy\nxLoVUzLPMmPNty2wwd1X904ws7cDawh36nMfzf8nYEny/3WEO/d7p12hHK8Cmxd6bF5EREREpBTd\n8ZZGdicwDNjfzJ4DtkumTWBT4/Ug4JGcO8S9nqRMZrYdoSfvEyk8jJQD25fIJk3sBxNiX1HhMrcH\nNgceK/BdoWm98u9iv5r8uzXw+gDzDaRUnuXGWsqBwP8DvkV4XL/X/iQ9n7t7j5ldQHnHvdzXFURE\nREREoqjhLY3sfmAt4V3pZ4GX3P0xM7sT+IqZbUZovF5bYN41FSy390mQKwhDSRXy1xJ5pIn9IDbF\nXo1lxuouMr2Snt6zyDPfK8BQMxvh7qsA3P1SADP7JDAr+X8bMAn4es68w+j/WkIaWwOrB3jUX0RE\nRESkKDW8pWG5+xtmdh+h8foMmxpMdwLtwDHADsAdlSymwLSXgdeANne/taxM42OvZJkvERr5uxX4\nbvfIvHJlcXe3GrH2vgu+K+GdbQCSx/L3IzzSD+GR8g3uvjBn3h3d/dmoiDct69Ey5hMRERER0Tve\n0vDuBN4DTE7+j7u/Qmh8fZvQOCznDmavVcm/o3snuHsPocfzo8xsfP4MSYdhaaSOvZJlJvPOBz5u\nZmNz5tkN+FDKWAvpt20qVaVY7ybcQZ+UN3034FV3732ne3dyHl9P3rdfn/z/LWb2z2Y2O/n7EDM7\nw8zazOyR5F30XB3AXSnjExERERHpQw1vaXR3Et4J3pG+Dew7CMNdPeXuL1SQ/wJCI+5cM/usmX06\naaCdCiwG7jWzmWZ2gpl9O7L37djYK1nmWYTHqO8ys2+Z2WnA7eTcES5DsW1TqbOoIFZ3fzJJe1je\nV88TOqbrfZLnXUBuZ2gnAlcm/98d+Auh0zqA/0Pokb0bmJdcIADAzCYC2wDXpYlPRERERCSfGt7S\n6O4ivDe8ktAjeK87CXeMK3nMHHe/HzgD2Ae4DLgK2M7dXwLeDVwKfAL4MXAy4e7vKVnEXsky3b2L\ncMd4GTAD+AKhgfsHwqPd0Yptm3LyyiDWS4GPmll7Tr7PAdOAS8zsLGAecKOZ/cjMpgEPJtsYd78N\nOJZNvbAfzKby+Evesv4ZeNrdb0+/liIiIiIim5i7OukVaVVm9mtgL3d/e71jKSUmVjMbCTwOnOLu\nl5W5vN8CnyYMM/YXd59gZqOA/d39xiTNZsBTwLnu/pNyliMiIiIiojveIi0if4xpM9sd+AhwW30i\nKq7SWN19JXABYfiwcl1FuOv9BeCWpEf0T/Q2uhPHE94Lr3RceBEREREZxHTHW6RFmNkLwGzgCWAX\n4CTCu9Qd7v54/SLrr5liFRERERGplIYTE2kdNwBTgLGEx6fvAr7ToA3ZZopVRERERKQiuuMtIiIi\nIiIikqFM73ib2bbAEYTOicrqWVlERKTJDSe8UnGTu79S51hERESkDrJ+1PwINo2bKyIiMpgdQ+jU\nT0RERAaZshreZvY14JuE9zMfBP7F3f9cIOlTEAYkHpP3xU2EVnm+SzixnJAa2I2EIYvztUXk0V2l\nWLJUbD1jxL32cAKXpE5bvf2qGuuZpWrsV42+jrGK7VeFj0KnR+xX5zTF8arVyrOYRl7PpcC1kNSJ\nIiIiMvhEN7zN7NPAD4ATgfuAqcBNZraHuy/NS74WQqN7XN4XwwtMCwpPbV7F1jRm02+oUixZKl6i\n6cU1vOOWVq39qhrrmaVq7FeNvo6xiu1Xhddzp6i8m2E7tVp5FtMU66lXrkRERAapcsbxngpc7O6/\ndPdFhGGAVhPGwhURERERaQpmNsnM/mRmr5tZt5ntU++YmpGZnWJmj0Sk/7KZPW1mw7KMS6SRRDW8\nkx/HROAPvdM8dIs+H9i/uqGJiIiIiGTDzIYC/wNsDfwr8Dng6boGlREz29/MppvZyAzy3go4BTg/\nYrbZwGbAl6sdj0ijir3jPYbwEumLedNfJLzvLSIiIiLSDN5GeMPoAnf/ubtf5e4r6h1URg4ApgGj\nM8j7i4T2wdyBEpnZqWZ2kZn9FmgHLgf+LYN4RBpS1r2aA6ELo+F500bVYsENYUK9A6gRrWfrGAzr\nCFrPVtMo6/kQsDBvml7tFmlQOyT/DtjYNrMt3H11DeLJkmWY93HAPHdfX3ThZocBxwJfAeYAWwJX\nA6eY2WR3vz3D+EQaQuwd76WErpB3yJu+A7Ck2ExHAFPyPh+OXHDz2rveAdSI1rN1DIZ1hMZpqGVt\nsJRno6zn3kBn3qdRe1sXGbzM7DLgdkIPnP9jZj1mdmvyOHaPmb3DzK4ys2XAnTnzvcnMLjWzJWa2\n1swWmtnxBfJPlW6A+Cab2f1mtsbM/mFmJ5rZWWbWk5NmJzObZWaLzGy1mS01s6vNbOe8vKYD30v+\nfCpZv24z26nSWM1sF2AfwmunAzkKuNPdb3f3ce7+grt3AcuAI9MsS6TZRd3xdvc3zGwBcCgwD8DM\nLPn7ouqHJyIiIiJSdT8FngNOB34E/Jnw6uRByfe/Av4OnEZyt9jMtgfuJdyEuohwQ+rDwC/MbCt3\nvygmXTFmth9wA/ACcCbhfP3MJJ/coTreBbyXcAf5OWAX4KvAbWa2l7v3Pm5zDbAH4d7XN4BXkukv\nVxor4RF2B7pSpPthgeldwIEl5hVpCeU8an4hMDtpgPcOJ7YFoZMEEREREZGG5u73mtlwQsP7Tne/\nFsDMehveD7j75/JmO5fQCH+nuy9Ppv3MzK4CzjKzi919XUS6Ys4mjPl5gLu/mMR1NbAoL91v3f2a\n3Almdj1wD+EO85XJui40sy5Cw/s37v5MTvofVxjrnsm/Txb60syOAT5KeBzoo2Y2GTg55136J4DP\nDpC/SMuIHk7M3a8GvgnMAB4gPF5yhLu/XOXYRERERERqzYGLC0z/JHA90GZm2/Z+gJsJnZZ1pEg3\nKiddP2Y2hPAk6XW9jW4Ad3+CcBecnGnrcuYbambbEBqyywdaRsQ6DRhrYltgQ7F34N39SuA84A3g\naHc/Nq8Du1eBzZOLICItrazO1dx9FjCryrGIiIiIiDSCPndwzWw7QuP6RAoPgeXA9mnTDbDc7YHN\ngccKfNdnWtJY/Q6hc7M3s6kDNSdFP8ZViDWtfYFF7r6hUBg5yxJpaTXp1fwSTgDG1WJRTaTQsWew\ni+twcwbTUqedxtmReU+PSt84tF/1F7dfnRSxX10QuV99q2n3KxGRQWdN3t+9T4leQRgGq5C/EobV\nSpOuGn5C6Cl8JuHx8hWEBux/k+6p1rTrNJBXgKFmNsLdVxVJsy/wlyLfbQ2sLvE4u0hLiG54m9nB\nwLeAiYTW9MfdfV61AxMRERERaRAvA68Bbe5+a7FEyaPiJdMN4CXCGIS7Ffhu97y/jwJmu/spOctv\np/BY3YXuKKdapxJ63zvflf5jKfbah7zH5HPsCjxa5rJFmkr0O97ACMJVq6+ix0JEREREpMW5ew+h\nd/CjzGx8/vdmNiYmXYnlzAc+bmZjc+bbjf5jE3bT/1z+ZDbddc/Vezd6Y6O80lgTdxMeLZs0QJp3\nUvyOdwdwV4rliDS96Dve7n4jcCNsHEpMRERERKTVnQpMBu41s0uAR4BtCE+BfgAYE5mumLOADwJ3\nmdl/Ec7Xv0a4o7xvTrrfAp8zs5XJMvYndMy2tECeCwgN5HPNbC6hs7N5lcbq7k+a2ULgMAqMcGRm\nOyb5PVjgu4nJd9cNtAyRVlGTd7xFRERERBpQ6qc33f0lM3s3MA34BPAVwjvODwOnxKYbYDldZvYh\n4PuEUYSeIzTG3558ep1M6NzlM8Bw4I+EBvBN+evl7veb2RnAScARhDvlu7r7M5XEmrgUONvM2gu8\nq90BLHT3VwrM98/A0+5+e8rliDQ1NbxFREREZNBx9/8l77Fsdz8bivec6e5LCQ3ek0vknSrdAPPf\nTt7j22b2a0IjvDfNSuBLBWZ/a5E8zyWMMV7VWAkN79MJFwAuS2I9m/Be9yTgd/kzmNlmwOcLxSPS\nqmrU8L6JcCEu14TkIyIi0ioeon//QmvrEYiINDEzG+7ua3P+3h34CEnDtpG4+0ozu4DQ+fJlybvh\npwH3AgcCXyww2/HAegqPly7SkmrU8D4CDScmIiKtb+/kk2sx8LM6xCIiTewJM5sNPAHsQnhEfC1w\nQR1jKsrdvwd8L/n/UjP7BaHRfaW7P1kg/cWo0S2DjB41FxERERFpLDcAU4CxwDpCz9/fcffH6xpV\nSu7+lXrHINJoyhnHewRhbMHeHs3famb7Asvc/dlqBiciIiIiMti4e6HHs0WkiZUzjvck4AHCsAQO\n/ADoYoCOKERERKR2zGxrM7vSzFaY2atm9vPkwnna+X9qZj1mVm5nSyIiIpKjnHG8/5fyGuwiIiJS\nG1cBOxDG9N2MML7uxcBnS81oZp8A3gM8XyLdtoROXJ5CPciJiMjgNJzQD8NNRYbN2yiq4W1mpxHG\n+NsTWEN43+Tb7v73EnOy6cn0UlIPpwjANGakTjuD6VF5S6NLu0/BvzMtKudzIx7g+I72q5biEfvV\nKZGjoFzOd1KnPVb7lZTJzPYkNIgnuvsDybR/AX5nZt909yUDzPtm4EfJ/L8vsagjgCurE7WIiEhT\nO4Zw0buo2DveBwM/Bu5P5j0PuNnM3uHua8oKUURERKppf+DV3kZ3Yj7hyvZ7gN8UmsnMDPgl8D13\nfzT8OaCnAK7YB/bcsu8XUx+Fme/oP8OkV+9PE/9GX310UulEiaH3Hx2V986Trk6ddlWR6VcSzrTy\nTeOE1HmfyCWp09bLjcCHarzMoyLS9usyu4ST+HaRb67pt+T7r/2PqLyf/2T6tD+4/5yovO+YlH4Y\n3hM5suD0epRlPRRbz+0i8sgfCLmUDxc45hVz+aNxeR87uvD0qa/DzC37T79+efq8R9y/f1Qsj026\nOyp9NTTyfrsUuDb896lSaaMa3u7+kdy/zew44CVgIvDHmLxEREQkE2MJdfNG7t5tZsuS74o5FVjv\n7j9JuZy1EBrdHaP6fjF6WP9pAKzrSJl18KaItMM6Yk6p4W0RaVcWmb4F4fnC/tIPodoMg60Op/Zx\n7hORdlh07jsVmb55v+86xsflvHVE2tEdu8ZlTvqGd7HyqkdZ1kOx9XxzRB6bRy6zY4v0aefH5l1k\nJx89pPB3D0XkvVXBg3VxxS5EZqlJ9tuSr1xVOpzYaMIV9GUV5iMiIiIDMLPzoOitOgj1ccQ9lz55\nTwROBvaLnXfqo6Ghnet5vfEtIiIt5iFgYd60mOqu7IZ38kjaD4E/uvsj5eYjIiIiqXwfuKxEmieA\nJcD2uRPNrA3YJvmukIMIT2E+m/OIeRtwoZn9q7u/tdgCZ76j/93tIxeUiFJERKTJ7J18ci0GfpZy\n/krueM8C9gIOLJ30Rvq/KTGB/qGLiIg0s0qvhxeX9JY6YI+pAGZ2NzDazH5PqKN7gHsIPVLeW2S2\nKwmV8sHAW4DXgS2BnwNpHz0XERGRIspqeJvZT4CPAAe7++LSc3yIpngyX0REpCKVXg+vnLsvMrNX\ngEOArxBeVfwx8Exuj+ZmtogwMslvgPXAjsB3gL8SXlW9A/i4u38jNoYpg6TKf2+9A6iR9G8WN7v0\nnfk1q8FSloNlPae01zuC2miV8oxueCeN7iOBQ9z9meqHJCIiIuVKhhMbA9xAaHD3AH8ADjezsTmN\n792BUQDuvpIwPFhuPkuBHc3sLe7+XEwMnTG9ojWxuL6Am9fgeT6x9Rveg6UsB8t6dsZ2vd6kWqU8\nY8fxngV0Ah8DVpnZDslXK9xdXamIiIjUX+9wYv/UOyF5x3stOcOJuXtbiXyOI7wrFjEwjYiIiBQy\nJDL9ScBI4HbghZxP3OCZIiIikpWCw4kRRiAZaDixjcysHTgfuMrdX696hCIiIoNM7DjesQ11ERER\nqYIshxPLW85Q4FdJfl8tlb7QcGJTxg2ex81FRGRwqNtwYnE8+ZQ2jRlROc9gWhnxyGDjWOlEOU6P\n2K/O5eyovL/D9Kj00ricdVHpj+WXEakfi8r7ochj597aD5tRlsOJ9abrbXTvCHwgzd3uQsOJiYiI\ntJqaDidmZicRekjdJZn0MDDD3W+MyUdERETilDGc2H7AAcA3gTcR6vyiV4uSRvdtwETCq2j3mtk5\n7n55FcIXEREZ1GIfHX+W8JhbB6FivhWYZ2Z7VTswERERiefui4CbgP8BLgSuIPTH8g/gKjMbA2E4\nMTM7Mvn/UKB33O+5wAeAS4Gfm9mHar4SIiIiLSaq4e3uv3P3G939cXd/zN3PAF4j9JIqIiIijeEz\nhM5QAb4OzAfeCawGvpBM3zicGPBm4DDAgGOBO4FzgTbQewkiIiKVKvsdbzMbQujNvJ1QQYuIiEhj\nWAWMBo5y93m9E81sPsnw07nDibn702Z2J7DA3f8tJ/1xwMxaBS0iItKqohveZjYBuBsYTrhyfrS7\nx/UCJCIiIlkaQ7hb/WLe9BeBtxeZZ2yR9CPNrN3d43oTFBERkY3KueO9CNiX8Hjap4C5ZnaIuz9Q\nfJabCO30XBOSj4iISKuodLCR5qPhxEREZDCo+XBi7r6BMFwJwANm9m5CT+cnFp/rCGBc7KJERESa\nTKWDjVTNUqAb2CFv+g4UH1JsSZH0Kwe6263hxEREZDCotIaP7dW8WB5tJVOJiIhITbj7G8ACYKqZ\nPWlma8zsHuDDwF1FZlsKnGBmL5nZCjO7C/g84fUyERERqUBUw9vMzjWzg81sZzObYGbnAe8jDFUi\nIiIijeNuYDJwM/BJQi/n2wHzAMzsPDPLHaN7OTCMMKzYUcAK4P3AdbULWUREpDXF3vHeHric8J73\nfMJY3ke4+23VDkxEREQqsj9wG+F9r2sJDemXgSOT78cCO/YmdvcvAh8kPEl3PbAb8Byh7hcREZEK\nRL3j7e5fyioQERERqQ4zG0a4OJ4/nNhsNg0ndnz+fO5+RzIfZmbAU8Cy7CMWERFpbWWP4w1gZqcC\n5wI/zB33s0DK5FPaDKbFRhGZPoanTjmNGVE5z2B6bDBSQx6xX50euc9ewNmp035L+0mLeTwi7Xui\nct4n8hjUPS79ftu2WPthEypnOLF83wJGAFdXMS4REZFBqeyGt5m9i9CT+YPVC0dERETqzcw+A5wJ\nfEX7W6EAACAASURBVMzdlw6UVsOJiYjIYFDz4cQAzGxLQodqXyJUzCIiItI4yhlODAAzm0IYHeVT\nafpw0XBiIiIyGNRrOLH/BK5391vLnF9EREQyUuZwYphZJ/AL4GzgejPrqknAIiIiLS664Z1cCX8n\ncFr1wxEREZEqiRpOLHm8/HLgdOCrwB3AUDMbWduwRUREWk/Uo+Zm9hbgh8BhydX0lG4EhudNm0D/\nm/UiIiLNrNI3wKoqdzixzwN/YdNwYt8jbzgx4ARCh2wXJn/vTOjB9IfAF2IWPOeFwfGO990kXcS3\nuIcYLGds9wOT6h1EpgZLWQ6W9ZyzFjrzm1gtqFXKM/Yd74mEq+VdyTAjECrp95nZ14F2dy/QFfiH\ngHEVhCkiItIMKn0DrDrKGU7M3d9vZscDXwYOIPThcqS7RzW6AeYuHhwN73sYHA3vhbTGSW9prd/w\nHixlOVjWc+66wdHwbpXyjG14z6f/es8GHgXOL9zoFhERkRqLHk7MzHYnDBF6kLv3bLq+LiIiIpWK\nani7+yrgkdxpZrYKeMXdH61mYCIiIlIbZjYEuBKY7u69A86nankXGk7s+bo9XS8iIpKNugwnlkd3\nuUVERBpL7HBiWxGesX2nmf1nMm0IYGa2Hvigu99eaEGFhhM7ckHZcYuIiDSkSl8mq7jh7e4fqDQP\nERERqR53f8PMFgCHsqkXc0v+vqjALCsJvZ7m+hrwfuAo4KkC8wwHWPR6/y+WvwFdKwrMsTpudLIX\nItIO7Xo5Ku+YE6BVRaavpvCGCadi6aRPWT9rqX2cf41I+2R07s8Umb6m33ddD8fl/HxE2uVdsZGP\nSJ2yWHnVoyzrodh6bojII/bV6a7V6dPG7CcAXUW6tF7eU/i7pyLyHlHwYF1cPfafRt5vl276b8ld\nxrJ8LdvMOoAFcCLZda6WPv5pzIjKeQbTM4kj0LtzjS3mcBv7TOVWEWk3j8z7pcj0UlvpjxM38u9R\nOX+IaZnFouNVpTZeD5/o7jUbF9vMjib0w3IScB8wFfgUsKe7v2xm5wFvcvdji8w/ndC5WkeR7z9D\neDxdRERksDvG3a8aKEHscGLToV9rdJG77xUbmYiIiGTH3a82szHADMIj5n8BjnD33lvD+cOJxboJ\nOIZwc0VvdYuIyGA0HNiFUCcOqJxHzRcSHlXrvQUS89SGiIiI1Ii7zwJmFfnu+ELTc74/Gzh7gO9f\nAQa8ui8iIjII3JUmUTkN7w05V8tFREREREREZABDyphndzN73sweN7MrzKySx9REREREREREWlps\nw/se4DjgCEJnLbsCd5hZ+m4WRURERERE5P+3d+fhcpVVvse/v4QhzCgIUUEBQZQLIkHAMBnFloC3\nQcWLDLaCIjK1XrqVSSQQFBQa9CKdbpwItCKde2kUFIii0BCZJAEkQlTGMCUQwETDnKz7x7sLKnWq\n6tSuU7vq1K7f53nqyTm73tq13rMq59Sq/Q42QHINNY+I6knjcyXdBjwM7A9c2PiR1zB0leitGboT\nmpmZWT+7m7QUSjWvO2ZmZjboRrSPd0QslvQnYPPmLSdT3HZiZmZmo8U2DP1Q+dXtxMzMzGxAtTPH\n+1WS1iQV3aN1T3MzMzPrIklHS3pQ0vOSbpG0Q69j6iRJUyQtr7nd0+u4RkLSbpKuyNbwWS5pnzpt\npkp6XNJzkn4laZiLLqPPcP2UdGGd3F7Vq3jbIelESbdJWiJpoaTLJb29Tru+zmcr/SxJPo+QdJek\nxdntJkmTa9r0dS5h+H6WIZeQs/CWdLak3SW9VdLOwOXAy8BPConOzMzM+oakTwDnAFOA7YC7gJnZ\nfuJlMpe0N/r47LZrb8MZsTVI+7wfBUTtnZKOB44BDgd2BJaS8rpKN4PsgKb9zFzNirk9sDuhdcxu\nwHeAnYAPAisDv5S0WqVBSfI5bD8z/Z7PR4DjgQnA9sBvgCskbQWlySUM089Mv+cy91DzjUh7dq4H\nPAXMAt6b7eVpZmZmg+1Y4IKIuBjSVQzgw8BngLN6GViHlWpr1Yi4hrQgD5JUp8kXgdMj4udZm08B\nC4GPADO6FedItdBPgBf7ObcRsXf195IOAZ4kFTOzssN9n88W+wn9n89f1Bw6WdKRpA8c7qEEuYSW\n+gl9nkvIecU7Ig6MiI0iYrWIeEtEHBQRDxYVnJmZmfUHSSuT3vT+unIsIgK4FpjYq7gKMjBbq0ra\nlHR1qTqvS4BbKV9eASZlQ5fnSZom6fW9DmiE1iVd3X8GSp3PFfpZpTT5lDRG0gHAqqRdpUqZy9p+\nVt3V97nMvbiapDcB3wT2AlYH/gwcGhFzOhxbqxG13HIqp+Q68ymcluPcU3Kd20a7PKsQ167YP5wl\nLbf8d76c68xH5HyNW7e1/vtqMl/Nee7/mbP9FjnafivnuW1ArQ+MJV1tqbYQ2LL74RSmsrXqH0kr\nx55KehO8dUQs7WFcRRlPKmjq5XV898Mp1NXAZcCDwNuAM4GrJE3MPkTqK9lV/W8DsyKictWwdPls\n0E8oST4lbQ3cTHrD+Rywf0TcL2kiJcplo35md5cil7kKb0nrAr8lfbKyJ7CI9O7t2c6HZmZmZja6\ntL+1qo12EVE9NPcPku4G7gcmAdf1JKiRmQZsBezS60AKVrefJcrnPGBbYB3g48Clkt7X25AKUbef\nEXFHWXKZd1XzE4D5EXFYRMyOiIcj4loPNzczMxt4i4BlpMVvqm0ILOh+ON0REYuBFrZW7VsLSMN1\nBiqvANn720X0YW4lnQ/sDUyKiOrdh0qVzyb9HKJf8xkRr0TEA1kB+hXSUPIjKVkum/SzXtu+zGXe\nwvvvgdslzcjG2M+RdFgRgZmZmVn/iIiXgdnAHpVj2RDQPYCbehVX0VTyrVWzN7gLWDGva5MWPSpt\nXgEkbURaULivcpsVo/sC74+I+dX3lSmfzfrZoH1f5rOOMcDYMuWygTGk6UtD9Gsu887x3oz0ycM5\nwNdJy9afJ+nFiPiPTgdnZmZmfeVcYLqk2cBtpFXOVwem9zKoTpJ0NnAlaXj5m4HT6POtVSWtQfrw\noLIQxWaStgWeiYhHSPNnT5Z0H/AQcDrwKPCzHoTbtmb9zG5TSPNIF2TtvkkazTBz6NlGJ0nTSNss\n7QMslVS5Gro4IioL2PR9PofrZ5brMuTzDNL85vnAWsDBwO7A17ImfZ9LaN7PsuQS8hfeY4DbIqKy\n6s9d2UT4I4CchffdwDY5n74fuZ/lMgj9HIQ+wuD0cyZpSY6yG5R8jm4RMSPbs3sqabjjncCe/b4F\nTI0ybq36HtI8ychu52THLwI+ExFnSVoduIC0evSNwF4R8VIvgh2BZv08CngX8ClSHx8n/QI9JRvN\n0S+OIPXt+prjhwIXA5Qkn8P1cxnlyOcGpNfnG4HFwO9Jv1Ovg9LkEpr0U9I4ypHL3IX3E8C9Ncfu\nBT7W/GHXMHTl5yUMxpukubifZTII/RyEPsLg9HNQCu/Rks+7SbFUy7NLQv+LiGmkxY5KKSIO7HUM\nnRYR/80w0w8j4lTSCu59q4V+Tu5WLEWJiJamkfZ7PofrZ3Z1vwz5HHZKb7/nEpr3syy5hPyF928Z\nuiXIlqThVk1MJn2AUa1vR2SZmZk1sA1DPwB4AvhuD2IxMzOz0SLv4mrfAt4r6URJb5N0EHAYcH7n\nQzMzMzMzMzPrf7kK74i4HfgoaTGDu4GvAF+MiEsLiM3MzMzMzMys7+Udak5EXAVc1WLzbGL3ojp3\nvUD3V4CPXK3zRdeodS/62QuD3M9Vcp7jxZZbzs/5mu1MDgY5l6NJ3tzPa3D8bw3uW5rj3KP551Qx\nmvP56t/A2sVOzMzMbEAoIu+buxwnT0PRf1zYE5iZmfWPgyPikl4HYWZmZt1XdOG9Hmk53YcYtGVd\nzczMknHAJsDMPt9yyszMzNpUaOFtZmZmZmZmNujyrmpuZmZmZmZmZjm48DYzMzMzMzMrkAtvMzMz\nMzMzswK58DYzMzMzMzMrkAtvMzMzMzMzswL1pPCWdLSkByU9L+kWSTv0Io6iSJoiaXnN7Z5exzUS\nknaTdIWkx7L+7FOnzVRJj0t6TtKvJG3ei1hHYrh+SrqwTm6v6lW87ZB0oqTbJC2RtFDS5ZLeXqdd\nX+ezlX6WJJ9HSLpL0uLsdpOkyTVt+jqXMHw/y5BLMzMzK6+uF96SPgGcA0wBtgPuAmZKWr/bsRRs\nLrAhMD677drbcEZsDeBO4ChgyB50ko4HjgEOB3YElpLyuko3g+yApv3MXM2KuT2wO6F1zG7Ad4Cd\ngA8CKwO/lLRapUFJ8jlsPzP9ns9HgOOBCcD2wG+AKyRtBaXJJQzTz0y/59LMzMxKquv7eEu6Bbg1\nIr6YfS/SG6rzIuKsrgZTEElTgH0jYkKvYymCpOXARyLiiqpjjwNnR8S3su/XBhYCn46IGb2JdGQa\n9PNCYJ2I+FjvIuus7EOvJ4HdI2JWdqyM+azXz9LlE0DS08CXIuLCMuayoqafpcylmZmZlUNXr3hL\nWpl0peLXlWORKv9rgYndjKULtsiGK98v6UeSNu51QEWRtCnp6lJ1XpcAt1K+vAJMyoYuz5M0TdLr\nex3QCK1Lurr/DJQ6nyv0s0pp8ilpjKQDgFWBG8qay9p+Vt1VmlyamZlZuazU5edbHxhLutpSbSGw\nZZdjKdItwCHAH4E3AqeS3gRvHRFLexhXUcaTCpp6eR3f/XAKdTVwGfAg8DbgTOAqSROj28NHOiAb\ncfJtYFZEVNYhKF0+G/QTSpJPSVsDNwPjgOeA/SPifkkTKVEuG/Uzu7sUuTQzM7Ny6nbhPRAiYmbV\nt3Ml3QY8DOwPXNibqKwTaobm/kHS3cD9wCTgup4ENTLTgK2AXXodSMHq9rNE+ZwHbAusA3wcuFTS\n+3obUiHq9jMi7ihRLs3MzKyEur242iJgGWnxm2obAgu6HEvXRMRi4E9A360k3KIFgBiwvAJExIOk\n13Xf5VbS+cDewKSIeKLqrlLls0k/h+jXfEbEKxHxQFaAfoU0lPxISpbLJv2s17Yvc2lmZmbl1NXC\nOyJeBmYDe1SOZUNA9wBu6mYs3SRpTdKbv6Zv+vtV9gZ3ASvmdW3SatKlzSuApI2A9eiz3GbF6L7A\n+yNifvV9Zcpns342aN+X+axjDDC2TLlsYAxp+tIQJcqlmZmZlUAvhpqfC0yXNBu4DTgWWB2Y3oNY\nCiHpbOBK0vDyNwOnAS8DP+llXCMhaQ3ShwfKDm0maVvgmYh4hDR/9mRJ9wEPAacDjwI/60G4bWvW\nz+w2hTSPdEHW7puk0Qwzh55tdJI0jbTN0j7AUkmVq6GLI+KF7Ou+z+dw/cxyXYZ8nkGa3zwfWAs4\nGNgd+FrWpO9zCc37WZZcmpmZWXl1vfCOiBnZtj5TScMd7wT2jIinuh1LgTYCLiFdbXkKmAW8NyKe\n7mlUI/Me0jzJyG7nZMcvAj4TEWdJWh24gLR69I3AXhHxUi+CHYFm/TwKeBfwKVIfHye9qT8lG83R\nL44g9e36muOHAhcDlCSfw/VzGeXI5wak1+cbgcXA70m/U6+D0uQSmvRT0jjKkUszMzMrqa7v421m\nZmZmZmY2SLq9uJqZmZmZmZnZQHHhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZm\nZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYg\nF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZm\nZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZm\nBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95m\nZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZm\nZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLh\nbWZmZmZmZlYgF95mZmZmZmZmBXLhbZaDpFMlLe9xDIdIWi7pLW0+/tTs8a/vVQydIOk4SfcUcN7P\nS3pY0sqdPreZmZmZDSYX3mb5RHbr5xhaerykiZKmSFq7gBhGRNJawHHANwo4/XRgFeDzBZzbzMzM\nzAaQC2+z/nMxsFpEzC/4eXYGTgHW7WEMjXwWGAtc2ukTR8SLwEXAP3X63GZmZmY2mFx4W1+TtHqv\nY+i2SF7qwlNpFMTQyCHAFQXGMAPYRNKkgs5vZmZmZgPEhbf1jaq5ye+UdImkZ4Abs/veJOmHkhZI\nekHSXEmH1jnHWyRNkzRP0nOSFkmaIemtddruKul3kp6X9GdJh7cY5zZZnP+z6tiE7NjtNW2vlnRz\n1ffD9qPR/GpJkyTdXh3vMHPSXydpuqRnJf0le95x2bmmAGdl7R7Knm9Z5TlrY6jKzdsanXOEsVY/\ndhPgXcC1w7VtV0TMAZ4B9i3qOczMzMxscKzU6wDMcqjMKf6/wJ+AEwFJ2gC4FVgGnAcsAvYCfiBp\nrYg4r+ocOwDvBX4CPApsAhwFXCdpq4h4gXTSrYGZwJOk4dYrA6dm3w9nLvAXYHfg59mx3YDlwLaS\n1oyIv0kSMBH49+w5W+3HkPnVkrYDrgYeB75K+r/91ewc9eZii3RV9wHgBGAC8DlgIenn+l/A24ED\ngC8CT2ePe6pBDJWva895WNU524211s5ZuzkttB2JOcAuBT+HmZmZmQ0AF97Wj+6IiH+ofCPp+6RC\n8t0R8Zfs8HclXQKcKumCbN4uwM8j4rLqk0m6ErgF2A/4cXb49OzfXSPisazdZaSiuqmICEm/JRXb\nFbsBl5OuoO4M/BJ4N7A22VV74Iwc/ah1GvAKsHNELMzinQHMaxLq7Ih49Sq+pPVJc6dPjIi7Jc0h\nFd4/yzGXu+E5RxhrtXdk/z5YfVDSJ4FtgZUj4n9nx3YFpkTE30lajZS/f4yIq1p4ngeAT7YYk5mZ\nmZlZQx5qbv0mgAtqjn0MuBIYK2m9yo1U3K5DuvKaHlxVuEpaKdtS6wHSFeoJ2fExwIeAyytFd/bY\nP5KugrfiRmBCVuwB7ApcBdzFawV55Sr4rLz9qJbFuwfw00ohm8X7AOnKcj31fo43AutJWrPFPuY+\nZ5ux1loPeCUinqsckLQl8DzpSn310PwPAwuyr18kXbnfpsXneRZYrd5QeTMzMzOzPFx4Wz969Uqn\npDeQVt0+nDQMuvr2w6zZBlXtx0maKmk+qRBbRBo+vk52A3gDsBpwX53n/mOLMd5IGp4+UdLbs3Pe\nCNzAioX3PRGxOG8/amzQJN56xypqr2I/m/37uiaPGc5w52w31uHsAlwBvJ80XL9iIvBbgIhYDpxd\nJ8ZGKovL9Xr7ODMzMzPrcx5qbv3o+aqvKx8e/Yi0BVQ9v6/6+nzg08C3SMPLF5MKq/+ksx9E3Q68\nQJrn/QjwZETcJ+lG4EhJq5Cugv9Xm/3ohGUNjjdczbxH56z1NLCSpDUiYilARPwQQNLHgGnZ12OB\n9wDHVD12ZV4b2j+c1wHPNRneb2ZmZmbWEhfe1u+eAv4KjI2I37TQfj9gekQcVzkgaVVW3Kv6KVJx\nv0Wdx7+jzrEhIuJlSbeRCu/5vFbs3QisChwMbEi6At5OP6o9SSryN69zX70+tKqIK72diLUyF3xT\nqubcZ8PytyMN6Yc0pPyViKiel79xRDzS4vNsCtzbYlszMzMzs4Y81Nz6WjZ8+DJgP0n/o/b+bHGv\nassY+rr/AjC25pwzgY9I2qjqXO8kzf1u1Y3ATsCk7Gsi4mlS4Xg8qbCtHM/bj1dlj702i3d81WM2\nBybniLfW0uzfdZu2yqFDsd5MuoL+nprjmwPPRkRlTvcWVA1fz+bbv5R9/QFJx0r6haSjs4Xkak0A\nbmoxJjMzMzOzhnzF28rgBFJxe6uk7wH3AK8Htgc+AFQXrT8H/kHSkqzdRNJiX4tqzjmFVAjOkjSN\nNET5GNIV1ne1GNeNwFeAjVlxePMNwOeBByPi8Tb7UetU0ocCN0n6N9L/7aOzeLdtMd5as0kF7hmS\nLgVeBq6IiOebP2xYI4o1Ih6UNBf4IDC96q7HSAvTrRQRr5C2jqteGO1wXlu1/gnSIm1/ioh/zVbG\nf5Wk7Uk/+5/m7p2ZmZmZWQ0X3tb3IuJJSTuS9tv+KHAkaR7wH4Djapp/gbSV1UGkomwWqYCbSdXQ\n6mw7rQ8B55K2v3o0O/+baL3wvol0hf1vpNXMK24kFYE3VDfO2Q9qHjtH0mTgX4CpWbynAltmt9wi\n4nZJJwNHAHuSRgpsSuuLkxUZ6w+B0yStWpmDHRGPSjoF+J6kh0mLrY2R9H9IP8cbIuLJrO29kr5A\nVojXmcf9v4CHI+L69ntqZmZmZpYowgv2mpWVpMuBrSKireK7m/LEKmlt4H7guIi4sM3nmwPsEBHL\nao6vAjwEnBER57dzbjMzMzOzap7jbVYStftNS9oC2Bu4rjcRNTbSWCNiCWlrsC+3+fzrAX+sLboz\nh5LmgtfuSW5mZmZm1hZf8TYrCUmPk+Y8PwBsQhoivjIwISLu711kQ/VTrGZmZmZmI+U53mblcTVw\nADAeeJE0x/ykUVrI9lOsZmZmZmYj4iveZmZmZmZmZgXyHG8zMzMzMzOzAhU61DxbwGhP0grBLxT5\nXGZmZqPUONJaBjMj4ukex2JmZmY90FbhLelo4Euk+Zl3Af8YEb+r03RPsn1yzczMBtzBwCW9DsLM\nzMy6L3fhLekTwDnA4cBtwLHATElvj4hFNc0fSv98DFi/5q5rgMl5n34UazRXfibp84cVfY7vtXzm\n73F4eyF1VSfyOTZn+3o7QRWt26/bfGswdOZ11Q//N1fP0fbFBsevIu1gVqsXr6tOaPT/p1E/X2n5\nzHleV9Cr31mdeN1unqPtGjnazge+Ca/+TTQzM7NB084V72OBCyLiYgBJRwAfBj4DnFXTNhtevj7w\nxpq7xtU51s8aFUj1+5mv5/3wc+pEPvO+HFsvHDqn26/bfIV3Z15X/fB/c60cbZ9vcHwc8KY6x3vx\nuuqERv9/GvXz5ZbPnP/V0IvXTydet5vlaLt2O0/gKVdmZmYDKtfiapJWBrYHfl05FmlZ9GuBiZ0N\nzczMzMxsKEmHSFou6S31vh+N+iHG0ULScZLu6eD5Pi/p4ayWMeuJvKuar08az7iw5vhC0nxvMzMz\nM7OiBSsOC6v9vickTZQ0RVK9YTGjIsZOGKafIz33WsBxwDc6eNrpwCrA5zt4TrNcCl3V/DXXkIYB\nVlvSnac2MzPrmuuyW7WlvQjEzHpjZ+AU4EKGvtm9GPhJRLzU9ag6r1k/R+qzpAt9l3bqhBHxoqSL\ngH8Czu/Uec3yyFt4LyKtPLRhzfENgQWNHzaZoXPv7s751P1q614H0CXuZ3kMQh8B3tXrALpkUPo5\nWl63789u1f4MHN2DWMysB9Tojmx6ZhmKbmjSzw44BLiigA8oZgDHSZoUEdd3+Nxmw8o11DwiXgZm\nA3tUjklS9v1N+Z56m3zN+9ZoeTNYtEHJ5yD0cxD6CINTkA5KPwfldWtWbpLeImmapHmSnpO0SNIM\nSW9t83zbSbpa0mJJf5V0raSd6rR7k6QfSHpM0guSHsjiWKnVuCRN4bWFhh/K5nMvG24eeisxSjo1\ne+zbJE2X9Kykv0j6oaTaYaWNfhaTJN0u6XlJf5Z0eOW8VW1G3M+qn+cPJS3Ifp5zJR3aQoybkP5w\nXdtKn/KIiDnAM8C+nT63WSvaGWp+LjBd0mxe205sddLcCTMzMzOzdu0AvBf4CfAosAlwFHCdpK0i\nouXdASRtBdwALCbNF36FNMf3ekm7R8TvsnZvBH5H2q7gAuCPwJuBj5Pe4y5pMa7LgLcDBwBfBJ7O\nQnkq+3fIHO9WY6x63AzgAeAEYAJwGGmtpROH+VlsB1wNPA58lVQDfJU0mrU6phH3U9IGwK2kUbLn\nZc+xF/ADSWtFxHlNQt05i2dOs/6MwBxgl4LObdZU7sI7ImZIWh+YShpifiewZ0Q81fyRZmZmZmZN\n/TwiLqs+IOlK4BZgP+DHOc71ddJ73V0i4uHsXP9BKqzP4rV5Id8ANgB2jIg7qh5/ap64ImKupDmk\ngvRnETG/gzFWzI6Iw6tiWJ80J7pp4Q2cRirqd46IhdljZwDzatp1op9nkIaivzsi/pId+66kS4BT\nJV0QES82iPMd2b8PNuqIpAOASaQPBh4FToiIdzRqX+MB4JMttjXrqLyrmgMQEdMiYpOIWC0iJkbE\n7Z0OzMzMzMwGS3VBJmklSa8nFUt/IV3hbYmkMcDfAZdXCtrs/AuAS4BdJa2ZTZnclzSn+I76Z+tc\nXO3EWB0G6Yp8tRuB9Wra1XuePYCfVoru7HkeIF0Fp+pYJ/r5MeBKYKyk9So34JfAOsOcZz3glYh4\nrkFfPg/8M3BURHwN+ADpinqrngVWa3V4vlkndWlV80GQb42JqZxSUBz97JVeBzAK5XtdnZ7jdXUG\np+U690lMydW+WH/N0Tbv39Z+fR3mjbv1rUzzvK4AZuR4be0/ql5XeRb9zDOn3Kuam7UqK4hOIi2w\n9WZe+0MYpKKtVW8gDRP/U5377iVdfNqYNEx6beAPXYqrnRjvrTpee3X52ezf1wF/a/A8GwCrAffV\nuW+FYyPtp6Q3AOsCh1N/667I4sktmxJwLmmkbWVe+lOkq94tn6YqDrOuyl14S9oN+DKwPWmp8o9E\nxBWdDszMzMzMBs75wKeBb5GGNy8mFUn/SZsjNUsW17IGxzu1yvhI+1lp8yPgogZtft/k8U8DK0la\nIyJqP7X8LPDXiJhVdWwXyHUl4XXAc02GupsVpp0r3muQ5nX/APivzoZjZmZmZgNsP2B6RBxXsYI1\nrgAAHAxJREFUOSBpVdJV1DyeAp4Dtqxz3zuB5cAjpCEpSxh+G5pW48pzJbXVGEfqSeAFYPM6921R\n8/1I+/kUaVja2Ij4TRuxVuacbwrMrblvZ+C3VXGNA7atPtaCTVlxBIFZ1+T+hC4iromIUyLiZxS7\nh5+ZmZmZDZZlDH1/+gVgbJ6TZEORfwnsW7PN1YbAgcCNEfG3bG/tnwJ/L6nZ3ONW46pcpR32g4JW\nYxzuPC0+z7XARySNr3qezYHJNc1H1M/suS4D9pP0P2pjyRaDa+ZmUn3xnjr3rcqKw/J3BOZFxJLs\n3O+QNFXS30u6TNK2dc4xgdxbIJt1hud4m5mZmdlo8XPgHyQtAe4BJpIWBsuzgFbFycAHgd9KmkYq\nKg8HVgGOq2p3EmmRsxskfZd0RfRNpO3EdskKu1bjmk0qHM+QdCnwMmnhtudHGONInQp8CLhJ0r+R\naoCjSVeVqwvUTvTzBNKq47dK+l52nteTpql+AGhYfEfEg5Lmkn4m02vuvoE0Lx5JqwCnkxaXQ9Lq\npAXpdouIpZJOzp73VZK2z+L4aaPnNyuSC28zMzMzGy2+QFot8iDS6pizSEXYTHIuiBUR92RrE51J\nKgbHkOYtH1S9I09EPC5pJ1IhdxBpsbXHgKtIQ8Fbjisibs+KviOAPbPn3JShi6LlinGkImKOpMnA\nv5C2BH6UVIxvyYpD3Ufcz4h4UtKOwCnAR4EjSXO3/0BrHyb8EDhN0qo1c7H/Bfi+pBOyWNYiK7xJ\nK6nfmRXd6wF/iYiXa877v4CHI+L6FmIw6zilETZtPlhaTpPF1bIhO7PhLQxdWXhr8q0KWzZ5fu4e\n0W+tUY7X1deZmuvco2tV8zzyrmr+QiFRjD6tf+4qat+7NPefOV5bo2tV8zwa/f2ald2qLSWbUrh9\nRMwpMiozszwkXQ5sFRH15pn3hKS1gfuB4yLiwgZt1gQWAG+LiIWSjgWWR8T/kfRJYBPg1oj4VdZ+\nFeAh4IyIOL8L3TAboktXvCeTFkA3MzMrs12zW7UH6OyIUTOz/CSNi4gXqr7fAtgbqFvc9kpELJF0\nNmkXpQsBJB0NTIiIz2bNvgRcWLUv+aXAcZL2AtbMbtXz0g8FXmLoPuhmXeOh5mZmZmZm5feApOmk\nTwM3IQ0TfwE4u4cx1RURZwFnVR1aA1hL0j+T5t8/GRH/WNX+CeDYJue7ABfd1mPt7OO9Bmk7gsr4\n582yVQOfiYhObHlgZmZmZmaddTVwADAeeJG0uvdJEXF/T6NqQVaIm/W13NuJkZb3v4O0mmEA5wBz\nyLd5vZmZmRVE0usk/VjSYknPSvp+9sF5q4//d0nLJX2hyDjNrHsi4rMRsVlErB4Rr4uID0fEXb2O\ny2xQ5L7iHRH/TXsFu5mZmXXHJcCGpG2AViFty3MB8MnhHijpo8BOpFWdm7Vbj7Sa8UMMzqqEZmZm\n1caRpm7MjIinmzXMtaq5pBNJ2wK8A3ieNETl+Ij4U4P22armh+PF1UYi38rzZ+RYUbh/V6q2kcqz\nAjrAd3Kugn5M37628qyC7lqjnjyvrT/nfF1t3pevqyeA70KXVjWX9A7S/rXbR8Qd2bE9gV8AG0XE\ngiaPfTNwM6mgvgr4VkSc16DtQcCPOxy+mZlZPzo4Ii5p1iDvFe/dgO8At2ePPRP4paR3RsTz7cVo\nZmZmHTQReLZSdGeuJX2KuxPws3oPkiTgYuCsiLg3fdvUQ5A2z12/5o5rSPuZlEWjj5Jmkj6hqPW9\n03Nsv/zV7+aMpvULGZ/LOQvwe3yuwT1De/rPfC/Xuf+aq3VvdPt1m3dD3+8dl+N1dVajdcTqv2oP\nz5nPscM3edWyXGfO93PJ85qF/P0c7Rq9ZnP9DMfl3C7+hTy/s16X69SH8826x+v1c79cZ4Y9G5y7\nnhM4vuW2C0hDysj+JjaTq/COiL2rv5d0CPAksD1DNy41MzOz7htP+tv8qohYJumZ7L5GTgBeyrHH\n7QuQiu7aUnBcnWP9rNGb2Ib93GRCjrPn/Um9tbAzN37E0J5unPPMz+aOpfu6/brNW3izcSdeV/V7\nmbffeQqIV3KeO9/Ppdh+jnaNXrO5foZj87yuaPCMjWzQkTPX6+e7cp0ZYLOWW74l97mBFoZBjnQ7\nsXVJuX1mhOcxMzOzJiSdCU0/hg/gnW2ee3vgC8B2eR97DUMnaCxpJwgzM7NR7HekYd/V8gz5brvw\nzoakfRuYFRH3tHseMzMza8m/ABcO0+YB0si3FS4zSBoLvD67r55dgTcAj1QNMR8LnCvpf0dEw0sF\nkxl6JeInwwRpZmbWb3bIbtXmA99o8fEjueI9DdgK2GX4pvU+D98a2GYET29mZjba3A3MrTnWmUX4\nstVSm66YCiDpZmBdSVeR/kYvB24BBNza4GE/Jv1R3g3YCPgbsCbwfaDVoedmZmbWQFuFt6Tzgb2B\n3SLiieEfUe/zcDMzs7LZhqEfKr+6qnlXRMQ8SU8D7wOOBFYjLYw6v3pFc0nzSDuT/Ax4iTRt9yTg\n96QVcW4APhIRX8wbw9Yj7kV/GJR+DkpP3csyGYx+DkYvy9PP3IV3VnTvC7wvIuZ3PiQzMzNrV7ad\n2PrA1aSCeznwa+DvJI2vKr63ANYBiIgl1CwBLGkRsLGkjSLi0TwxDMp4trK8GRzeYPR0MF63g5HL\nQennYLxmy9PPXIW3pGnAgcA+wFJJG2Z3LY4Ib2hrZmbWe5XtxD5cOZDN8X6Bqu3EImK4HYEOIc0V\n+0sxYZqZmQ2OMTnbHwGsDVwPPF5127+zYZmZmVmb6m4nRtqBpNl2Yq+StCppvZhLIuJvHY/QzMxs\nwOTdxztvoW5mZmYdUOR2YjXPsxLwf7PzHTVcey+famZmg6Bn24nZSLW+tf0ZTM115pM4JW8wNoAC\nDd+oyj/mfF3N4LSW2+7PlFznLlaeWTO15UYnz92/8ry2Ns/9+6r1353LLs33u3PsAXleh3n+fA43\nortlRW4nVmlXKbo3Bj7QytVuL59qZmaDoKvbiUk6grRC6ibZoT8AUyPimjznMTMzs3za2E5sO2Bn\n4EvAm0h/819s8riVgOuA7UlT0W6V9PWIuKgD4ZuZmQ20vEPHHyENc5tA+sP8G+AKSVt1OjAzMzPL\nLyLmATOB/wecC/yItB7Ln4FLJK0PaTsxSftmX68EVPb9vhT4APBD4PuSJne9E2ZmZiWTq/COiF9E\nxDURcX9E3BcRJwN/Ja2SamZmZqPDQaTFUAGOAa4F3g08B3wmO/7qdmLAm4EPAgI+DdwInEEaJz+a\n5oKYmZn1pbbneEsaQ1rNfFXSH2gzMzMbHZYC6wL7RcQVlYOSriVtN7bCdmIR8bCkG4HZEfFPVe0P\nAb7VraDNzMzKKnfhLWlr4GbSqkLPAftHxH2dDszMzMzatj7pavXCmuMLgS0bPGZ8g/ZrS1o1IhrO\nDzczM7Pm2rniPQ/YljQ87ePApZLeFxF3NH6INxsxM7NB8PvsVq3cq9n7L7yZmQ2Crm8nFhGvkLYr\nAbhD0o6klc4Pb/wobzZiZmaD4F3ZrdrjwL91O5BFwDJgw5rjG9J4S7EFDdovaXa123/hzcxsEIx0\nO7G8q5o3OkfHNik1MzOzkYmIl4HZwLGSHpT0vKRbgL2Amxo8bBHwOUlPSlos6SbgU6TpZWZmZjYC\nuQpvSWdI2k3SWyVtLelMYHfSViVmZmY2etwMTAJ+CXyMtMr5G4ArACSdKal6j+6/ACuTthXbD1gM\nvB/4afdCNjMzK6e8V7w3AC4izfO+lrSX954RcV2nAzMzM7MRmQhcB+wJ/BepkH4K2De7fzywcaVx\nRHwW+BBpevaVwObAo6S//WZmZjYCueZ4R8RhRQViZmZmnSFpZdKH47XbiU3nte3EDq19XETckD0O\nSQIeAp4pPmIzM7Nya3sfbwBJJwBnAN+u3vfTWqGWW57EKYWdO79oueXZTM115i8zJW8w1kWR83X1\niRyv25s4Lde5dx41r5W8q1XXrv3cyXP3q7y/r1r/s7XSgfl+d85ZdknLbSeM/XSOMxf5O7mhdrYT\nq/VlYA1gRgfjMjMzG0htF96SdiCtZH5X58IxMzOzXpN0EPBVYJ+IWNSsrbcTMzOzQdD17cQAJK1J\nWlDtMNIfZjMzMxs92tlODABJBwDfBT7eyhou3k7MzMwGQa+2E/tX4MqI+E2bjzczM7OCtLmdGJIO\nBH4AnAZcKWlOVwI2MzMrudyFd/ZJ+LuBEzsfjpmZmXVIru3EsuHlFwFfAY4CbgBWkrR2d8M2MzMr\nn7z7eG8EfBs4OPs03czMzEanXNuJAZ8jLch2LvBW0h7e25D+7udyd9sh95e5vQ6gawajp4Pxuh2M\nXA5KPwfjNVuefuad47096dPyOdk2I5D+SO8u6Rhg1Yios+y1l14xM7NBcBdD3yJ0f4X6drYTi4j3\nSzoU+DywM2kNl30j4jN5n38ug/EXfi7p3Uz5DUZPB+N1Oxi5HJR+DsZrtjz9zFt4X8vQfk8H7gW+\nUb/oBi+9YmZmg2Hb7FbtcWBatwPJvZ2YpC1IW4TuGhHLX/t83czMzEYqV+EdEUuBe6qPSVoKPB0R\n93YyMDMzM+sOSWOAHwNTIuL+yuFWHltvTNuSDsZmZmY2GvRkO7EaDa5ym5mZWY/k3U5sLeA9wLsl\n/Wt2bAwgSS8BH4qI6+s9Ub0xbT9pM2gzM7PRaqTbiY248I6ID4z0HGZmZtY5EfGypNnAHry2irmy\n78+r85AlDJ0QeTRpgbX9gIfqPGYcpAq/1gvAE+0EPko1usLQsJ8P5dmFrbifVP4zN3rE0J4+kvPM\nf80dS/d1+3Wb+8rVI514XdXvZd5+j83RdlnOc+f7uRTbz9Gu0Ws2189wWd5dI/P8FPOtcZInm7/P\ndWaAB1puOT/HWas+ya4d/DWEGk7L7gBJE4DZcDie491Nref0DKbmOvNJTCkkjsTzCUe3YX+f1Mjz\nyzbvufPsbvRkznNbd+X7PbHs705vue3YX51SYCx5fl89AXwXYPuI6Nq+2JL2J63DcgRwG3As8HHg\nHRHxlKQzgTdFxKcbPH4KaXG1CQ3uP4g0PN3MzGzQHRwRlzRrkOuKd/ZHuLbymhcRW+WNzMzMzIoT\nETMkrQ9MJQ0xvxPYMyKeyprUbieW10zgYNLV8O4v3W5mZtZ744BNSH8Tm2pnqPlc0lC1ysf9r7Rx\nDjMzMytYREyjwZLqtduJ1bn/NOC0Jvc/DTT9dN/MzGwA3NRKo3YK71eqPi03MzMzMzMzsybGtPGY\nLSQ9Jul+ST+SNJJhamZmZmZmZmallrfwvgU4BNiTtFjLpsANktbocFxmZmZmZmZmpZBrqHlEVE8a\nnyvpNuBhYH/gwsaPvIahqxZvDWyT5+nNzMxGubtJS6FU87pjZmZmg25E+3hHxGJJfwI2b95yMt5O\nzMzMym8bhn6o/Op2YmZmZjag2pnj/SpJa5KK7rLtR29mZmZtkHS0pAclPS/pFkk79DqmTpI0RdLy\nmts9vY5rJCTtJumKbA2f5ZL2qdNmqqTHJT0n6VeShrnoMvoM109JF9bJ7VW9ircdkk6UdJukJZIW\nSrpc0tvrtOvrfLbSz5Lk8whJd0lanN1ukjS5pk1f5xKG72cZcgk5C29JZ0vaXdJbJe0MXA68DPyk\nkOjMzMysb0j6BHAOMAXYDrgLmJntJ14mc0l7o4/Pbrv2NpwRW4O0z/tRQNTeKel44BjgcGBHYCkp\nr6t0M8gOaNrPzNWsmNsDuxNax+wGfAfYCfggsDLwS0mrVRqUJJ/D9jPT7/l8BDgemABsD/wGuELS\nVlCaXMIw/cz0ey5zDzXfiLRn53rAU8As4L3ZXp5mZmY22I4FLoiIiyFdxQA+DHwGOKuXgXVYqbZW\njYhrSAvyIEl1mnwROD0ifp61+RSwEPgIMKNbcY5UC/0EeLGfcxsRe1d/L+kQ4ElSMTMrO9z3+Wyx\nn9D/+fxFzaGTJR1J+sDhHkqQS2ipn9DnuYScV7wj4sCI2CgiVouIt0TEQRHxYFHBmZmZWX+QtDLp\nTe+vK8ciIoBrgYm9iqsgA7O1qqRNSVeXqvO6BLiV8uUVYFI2dHmepGmSXt/rgEZoXdLV/Weg1Plc\noZ9VSpNPSWMkHQCsStpVqpS5rO1n1V19n8vci6tJehPwTWAvYHXgz8ChETGnw7FZ2xp9iDvUVzgl\n15nP5rSW236ZKbnObaNdkSszP5+r9bOrnNRy29e9lO81PnrU7gQxnH5dObv131cAY3/11Rytd8wX\nChvkaHttjrb9NuKvbesDY0lXW6otBLbsfjiFqWyt+kfSyrGnkt4Ebx0RS3sYV1HGkwqaenkd3/1w\nCnU1cBnwIPA24EzgKkkTsw+R+kp2Vf/bwKyIqFw1LF0+G/QTSpJPSVsDN5PeGDwH7B8R90uaSIly\n2aif2d2lyGWuwlvSusBvSZ+s7AksArYAnu18aGZmZmajS/tbq9poFxHVQ3P/IOlu4H5gEnBdT4Ia\nmWnAVsAuvQ6kYHX7WaJ8zgO2BdYBPg5cKul9vQ2pEHX7GRF3lCWXeVc1PwGYHxGHRcTsiHg4Iq71\ncHMzM7OBtwhYRlr8ptqGwILuh9MdEbEYaGFr1b61gDQ0ZaDyCpC9v11EH+ZW0vnA3sCkiKjefahU\n+WzSzyH6NZ8R8UpEPJAVoF8hDSU/kpLlskk/67Xty1zmLbz/Hrhd0oxsjP0cSYcVEZiZmZn1j4h4\nGZgN7FE5lg0B3QO4qVdxFU0l31o1e4O7gBXzujZp0aPS5hVA0kakBYX7KrdZMbov8P6ImF99X5ny\n2ayfDdr3ZT7rGAOMLVMuGxhDmr40RL/mMu8c781InzycA3ydNIHuPEkvRsR/dDo4MzMz6yvnAtMl\nzQZuI61yvjowvZdBdZKks4ErScPL3wycRp9vrSppDdKHB5VFFzaTtC3wTEQ8Qpo/e7Kk+4CHgNOB\nR4Gf9SDctjXrZ3abQppHuiBr903SaIaZQ882OkmaRtpmaR9gqaTK1dDFEVFZDKTv8zlcP7NclyGf\nZ5DmN88H1gIOBnYHvpY16ftcQvN+liWXkL/wHgPcFhGV1W3uyibCHwHkLLzvBrbJ+fT9aDD6eQdp\nw9byG4R8DkIfwf0sm7nA1r0OYuBFxIxsz+6ppOGOdwJ79vsWMDXKuLXqe0jzJCO7nZMdvwj4TESc\nJWl14ALS6tE3AntFxEu9CHYEmvXzKOBdwKdIfXyc9Kb+lGw0R784gtS362uOHwpcDFCSfA7Xz2WU\nI58bkF6fbwQWA78n/U69DkqTS2jST0njKEcucxfeTwD31hy7F/hY84ddw9AVepcwOG8Gy9/POxmU\nwnsQ8jkIfQT3s2xGS+F9J+n9QrV8q/b3u4iYRlrsqJQi4sBex9BpEfHfDDP9MCJOJa3g3rda6Ofk\nbsVSlIhoaRppv+dzuH5mV/fLkM9hp/T2ey6heT/LkkvIX3j/lqFbgmxJGm7VxGTSBxjV+nZElpmZ\nWQPvzm7VHgPO70EsZmZmNlrkXVztW8B7JZ0o6W2SDgIOw+8ozMzMzMzMzOrKVXhHxO3AR0mLGdwN\nfAX4YkRcWkBsZmZmZmZmZn0v71BzIuIq4KoWm2cTuxfVuesF+mwF+DaN7n4Gkav9ow2OP1/3vtHb\n7/aN7nx2Ri/6mO91eNfyPO0b9WW053KVnO0braMy2vuZV6Pc1+vnfTnP/UyOto/laPtk5YvaxU7M\nzMxsQCgi3xveXCdPQ9F/XNgTmJmZ9Y+DI+KSXgdhZmZm3Vd04b0esCdpX7kXmrc2MzMrpXHAJsDM\nPt9yyszMzNpUaOFtZmZmZmZmNujyrmpuZmZmZmZmZjm48DYzMzMzMzMrkAtvMzMzMzMzswK58DYz\nMzMzMzMrkAtvMzMzMzMzswL1pPCWdLSkByU9L+kWSTv0Io6iSJoiaXnN7Z5exzUSknaTdIWkx7L+\n7FOnzVRJj0t6TtKvJG3ei1hHYrh+SrqwTm6v6lW87ZB0oqTbJC2RtFDS5ZLeXqddX+ezlX6WJJ9H\nSLpL0uLsdpOkyTVt+jqXMHw/y5BLMzMzK6+uF96SPgGcA0wBtgPuAmZKWr/bsRRsLrAhMD677drb\ncEZsDeBO4ChgyB50ko4HjgEOB3YElpLyuko3g+yApv3MXM2KuT2wO6F1zG7Ad4CdgA8CKwO/lLRa\npUFJ8jlsPzP9ns9HgOOBCcD2wG+AKyRtBaXJJQzTz0y/59LMzMxKquv7eEu6Bbg1Ir6YfS/SG6rz\nIuKsrgZTEElTgH0jYkKvYymCpOXARyLiiqpjjwNnR8S3su/XBhYCn46IGb2JdGQa9PNCYJ2I+Fjv\nIuus7EOvJ4HdI2JWdqyM+azXz9LlE0DS08CXIuLCMuayoqafpcylmZmZlUNXr3hLWpl0peLXlWOR\nKv9rgYndjKULtsiGK98v6UeSNu51QEWRtCnp6lJ1XpcAt1K+vAJMyoYuz5M0TdLrex3QCK1Lurr/\nDJQ6nyv0s0pp8ilpjKQDgFWBG8qay9p+Vt1VmlyamZlZuazU5edbHxhLutpSbSGwZZdjKdItwCHA\nH4E3AqeS3gRvHRFLexhXUcaTCpp6eR3f/XAKdTVwGfAg8DbgTOAqSROj28NHOiAbcfJtYFZEVNYh\nKF0+G/QTSpJPSVsDNwPjgOeA/SPifkkTKVEuG/Uzu7sUuTQzM7Ny6nbhPRAiYmbVt3Ml3QY8DOwP\nXNibqKwTaobm/kHS3cD9wCTgup4ENTLTgK2AXXodSMHq9rNE+ZwHbAusA3wcuFTS+3obUiHq9jMi\n7ihRLs3MzKyEur242iJgGWnxm2obAgu6HEvXRMRi4E9A360k3KIFgBiwvAJExIOk13Xf5VbS+cDe\nwKSIeKLqrlLls0k/h+jXfEbEKxHxQFaAfoU0lPxISpbLJv2s17Yvc2lmZmbl1NXCOyJeBmYDe1SO\nZUNA9wBu6mYs3SRpTdKbv6Zv+vtV9gZ3ASvmdW3SatKlzSuApI2A9eiz3GbF6L7A+yNifvV9Zcpn\ns342aN+X+axjDDC2TLlsYAxp+tIQJcqlmZmZlUAvhpqfC0yXNBu4DTgWWB2Y3oNYCiHpbOBK0vDy\nNwOnAS8DP+llXCMhaQ3ShwfKDm0maVvgmYh4hDR/9mRJ9wEPAacDjwI/60G4bWvWz+w2hTSPdEHW\n7puk0Qwzh55tdJI0jbTN0j7AUkmVq6GLI+KF7Ou+z+dw/cxyXYZ8nkGa3zwfWAs4GNgd+FrWpO9z\nCc37WZZcmpmZWXl1vfCOiBnZtj5TScMd7wT2jIinuh1LgTYCLiFdbXkKmAW8NyKe7mlUI/Me0jzJ\nyG7nZMcvAj4TEWdJWh24gLR69I3AXhHxUi+CHYFm/TwKeBfwKVIfHye9qT8lG83RL44g9e36muOH\nAhcDlCSfw/VzGeXI5wak1+cbgcXA70m/U6+D0uQSmvRT0jjKkUszMzMrqa7v421mZmZmZmY2SLq9\nuJqZmZmZmZnZQHHhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZm\nZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZm\nBfr/2cIPTHPBRAwAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "fig, ((ax1,ax2),(ax3,ax5),(ax4,ax6),) = plt.subplots(nrows=3, ncols=2)\n", + "plt.rcParams['savefig.facecolor'] = \"0.8\"\n", + "fs = 12 # font size\n", + "fig.set_figwidth(10)\n", + "fig.set_figheight(5)\n", + "\n", + "ax1.imshow(_X.T - _y.T, interpolation='none') ; ax1.set_title('input ($X$) and target ($y$)')\n", + "ax2.imshow(_y_hat[0,:,:].T, interpolation='none') ; ax2.set_title('prediction ($\\hat y$)')\n", + "\n", + "ax3.imshow(_w_w[0,:,:].T, interpolation='none') ; ax3.set_title('write weighting ($w_w$)')\n", + "ax4.imshow(_w_r[0,:,:,0].T, interpolation='none') ; ax4.set_title('read weighting ($w_r$)')\n", + "\n", + "ax5.imshow(_f[0,:,:].T, interpolation='none') ; ax5.set_title('free gate ($f$)') ; ax5.set_aspect(3)\n", + "ax6.imshow(_g_a[0,:,:].T, interpolation='none') ; ax6.set_title('allocation gate ($g_a$)') ; ax6.set_aspect(3)\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/repeat-copy/.ipynb_checkpoints/repeat-copy-rnn-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/repeat-copy-rnn-checkpoint.ipynb new file mode 100644 index 0000000..a6cca09 --- /dev/null +++ b/repeat-copy/.ipynb_checkpoints/repeat-copy-rnn-checkpoint.ipynb @@ -0,0 +1,405 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Differentiable Neural Computer (DNC) using a RNN Controller\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from rnn_controller import RNNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 10000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"nn_models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAADyNJREFUeJzt3W2sHNV5wPH/cx24TiAY1U7BKmkgJSmyHdGAQ0oTXhrb\nckRVaKWK4CJVQCIEJP3gqi1BRiKlhKSNiGmhjlq14FQE01RNVKIGjA20KSKUxk5ofA1F4SXQYlMM\nwpGMTY3v6YfZm96X3Xv37J3Z3dn9/6T9sLOzs8+Z5+ycnZ2ZZyKlhCRJqq+RXgcgSZLmx8FckqSa\nczCXJKnmHMwlSao5B3NJkmrOwVySpJpzMJckqeYczCVJqjkHc0mSau5tVS48IhYDa4HngUNVfpYk\nSQNmIXAysDWl9OpsM3Y0mEfEp4HfB04EngB+N6X0701mXQt8rZPPkCRJAFwK3D3bDNmDeUR8ArgF\nuBJ4HFgPbI2I96eU9k2b/XmAi+76dZactnjKC9vWP8iajatmLH/Bys1Z8eRWlh//3mVZ849kxjO+\n4/Km07et386ajauzltVMbin9O1YelfsJWXNf8b23mk5v1d47Vrbf5T65o/myS5O5Mlv1ha0Uv1qn\ny+1rlff9Fn0zV6vc5vfNvM1Pq77Wevk5fb9V8M2zm903S+prrWTntkU8rbbLlceTqby+Vk5+q9su\n7wO+AY2xdDad7JmvB/4ypfS3ABFxFfBrwBXAn06b9xDAktMWs/SME6e8MHr86IxpAAsyg8neoDX5\nzNnknlTQavmji5q3N1f+fXGOzv2ErLmXnnG46fSFLfIL7W9gWy27NLkb2BbTFwJLm0zP7Wv91vdb\nadWX8/tm3g/N/P6Q0/dbBd88u9mxlNTXWsnObYt4Wm2XK48nU3l9rZz8Vr9dnvswdVaOIuIo4Ezg\nwYlpqbjt2nbg7NzoJEnS/OX+4FpCsQPx8rTpL1McP5ckSV3mpWmSJNVc7jHzfcAR4IRp008A9rZ6\n07b1DzJ6/OiUaYt+/rjMj6635euW9TqErlp+yfC0d0WvA+iy4erLw5XdYfreFvopvz8Edk2b1v4V\n3VmDeUrpcETsAFYB9wJERDSe/3mr963ZuKqUk7/qbMW65b0OoauWD1F7+2lz0A3D1ZeHK7srhuqH\nGvRXfj/QeEy2B/irtt7dydnsXwY2Nwb1iUvT3gFs7mBZkiRpnrIH85TS1yNiCXAjxd/rPwDWppRe\nKTs4SZI0t44qwKWUNgGbSo5FkiR1wLPZJUmquU7KuZ4D/AFF8ZilwG+klO4tK6DsqlZHPps1/8jI\nF7Pmj6y5YWTkC1nzj49fl/kJufLW6IYjeZWPPj+SWy62/TV6U+ayrx+vtmJcfl/I62vDJ2+N5ve1\nIZJbYc6+Oavcim790Dc72TM/huI4+TXkj72SJKlknZwAdz9wP/z0sjRJktRDHjOXJKnmHMwlSao5\nB3NJkmquo+vMczWrzb78kmVDWDpQkqRmulibvVPWZpckaTZdrs0eEccAp/L/F4m+NyJOB15LKb2Y\nuzxJkjQ/neyZrwQeprjGPAG3NKZ/FbiipLgkSVKbOrnO/F/wxDlJkvqGg7IkSTWXtWceEdcBvwmc\nBhwEHgWuTSk9Pccbi0cbxsfzaq1X7UifxZNbc29DxfXKc20Y/9/qFt5n9anrXh6x/+pT99Eaze1r\nC/L6WtXbwdw1WXXd7uy+tiCvr1W/Hex938zdMz8HuA34MLAaOAp4ICLeXnZgkiSpPVl75imlCyY/\nj4jLgP+huIPaI+WFJUmS2jXfY+bHU/wD81oJsUiSpA50PJg37ph2K/BISml3eSFJkqQc86kAtwlY\nBnykpFgkSVIHOhrMI+J24ALgnJTSnrnm37Z+O6OLptVmX7eMFeuWd/LxkiQNmC7XZm8M5BcB56WU\nXmjnPWs2rrY2uyRJLXWxNntEbALWARcCByLihMZL+1NK7f+EkCRJpck9Ae4q4Djgn4GXJj0uLjcs\nSZLUrtzrzC3/KklSn3FwliSp5nKPmV8FXA2c3Jg0BtyYUrq/tIhyi49nFvXNraCbX1P5usxPqFbF\nq5PcNZpTU/n63HrKFReQPnIkr172gsy+M2w2HMmr0//5BUdXFEn1IrNvZt83IPN73m/3nMiWMrc7\nufcNyL4HRjX3nNiz8zB3rGxv3tw98xeBa4EzKEq4PgTcGxHLMpcjSZJKknvM/J+mTbo+Iq6muPGK\nVeAkSeqBjivARcQIxVnso8C/lhaRJEnK0knRmBXAd4GFwBvAxSmlH5UdmCRJak8nZ7M/BZwOnAXc\nDtwTER8sNSpJktS27D3zlNJbwLONp9+PiLMoznC/stV7rM0uSVJrY1vGGLtn6qlnh15/s+33z+eu\naRNGgAWzzWBtdkmSWlu+bjnLp+3g7tm5lztW3tnW+3OvM78ZuA94AXgncClwLnBTznIkSVJ5cvfM\nfxb4KrAU2A/8B7A2pfRw2YFJkqT25F5n/qmqApEkSZ2xNrskSTU3rxPgIuKzwM3ArSml32s5Y0pt\nF/2uuhZ6bn3t3Hiqlls7PacWOuTXQ8+vr51ZIzlHZiH68brXp65Ybl3/Dbm19Ossc+Vk10LP/J7n\n3geg3+45kd/XMmuhZ2838+4DUNV2Nme9dLxnHhEforgc7YlOlyFJkuavo8E8Io4F7gI+BbxeakSS\nJClLp3vmfwF8K6X0UJnBSJKkfJ3UZr8E+CWgzbusSpKkKuUWjTkJuBVYnVIaorNdJEnqX7l75mcC\n7wJ2Rvz0PLsFwLkR8RlgNKWZ51tvW/8go8dPq81+yTJWrFvWQciSJA2WXVvGGNsytTb7m/urq82+\nHfjAtGmbgSeBLzYbyAHWbFxlbXZJklpYsW75jJuP7dm5l785s4La7CmlA8CUnw4RcQB4NaX0ZM6y\nJElSOcqoAJd5Ob4kSSrTvG+BmlL6WBmBSJKkzkSLw9zlLDziDGDHJ3dc3v4x89x4cusA5qpw/UD1\nZRXrvDqrjqVyFfed3P/ERt7WXyU8c1W9OnP0Xd/sp5VD/5WLzVX16my3/0w6Zn5mSmnnbPNm/c0e\nETdExPi0x+653ylJkqrSyd/su4BVwMRvi7fKC0eSJOXqZDB/K6X0SumRSJKkjnRyNvv7IuK/I+KZ\niLgrIt5delSSJKltuYP5Y8BlwFrgKuAU4DsRcUzJcUmSpDblFo3ZOunproh4HPgxcDHQskzNtvXb\nGV00tZzrovccx8dvW5vz8bW2a8vuoSpfu2vL2IxqRoNqmNoKw9XesS1jLB+StoLbqV7qdjnXKVJK\n+yPiaeDU2eZbs3H1jEvT/u7Cv5/PR9fO2D3D9SUZ27K7b74kVRumtsJwtXfsnt1DNZi7neqd+ZZz\nnVcFuIg4lmIg3zOf5UiSpM7lXmf+pYg4NyLeExG/AnwTOAxsqSQ6SZI0p9y/2U8C7gYWA68AjwC/\nnFJ6tezAJElSe3JPgFuXufyFAPue3DfjhTf3v8menXubfUjeJ/RT/dFZvPl68/aOZC5+vNk6m0Wv\nVmer/A5iOdfS+nKu3HKuJfW1lu2tWC8qlh5q8b3tu75Z8XYqV9XbtbKUsZ3qRLv9Z9LYuXDOZVZc\nm/23ga9V9gGSJA2+S1NKd882Q9WD+WKKa9KfBw5V9kGSJA2ehcDJwNa5DmdXOphLkqTqzevSNEmS\n1HsO5pIk1ZyDuSRJNedgLklSzfVkMI+IT0fEcxFxMCIei4gP9SKOqkXEDRExPu2xe+539r+IOCci\n7m3cDnc8Ii5sMs+NEfFSRLwREdsiYtYa/v1srvZGxJ1Ncv3tXsU7HxFxXUQ8HhE/iYiXI+KbEfH+\nJvMNRH7bae+g5DciroqIJyJif+PxaER8fNo8A5FXmLu9g5JX6MFgHhGfAG4BbgA+CDwBbI2IJd2O\npUt2AScAJzYeH+1tOKU5BvgBcA1NypdExLXAZ4ArgbOAAxR5PrqbQZZo1vY23MfUXOcWWeoX5wC3\nAR8GVgNHAQ9ExNsnZhiw/M7Z3oZByO+LwLXAGcCZwEPAvRGxDAYurzBHexsGIa+QUurqg+Ke6H82\n6XkA/wX8Ybdj6UJbbwB29jqOLrRzHLhw2rSXgPWTnh8HHAQu7nW8FbX3TuAbvY6tovYuabT5o0OS\n32btHeT8vgpcPuh5bdHegclrV/fMI+Ioil9HD05MS8Ua3Q6c3c1Yuuh9jb9mn4mIuyLi3b0OqGoR\ncQrFL9zJef4J8G8Mbp4Bzm/8TftURGyKiJ/pdUAlOZ7i34jXYCjyO6W9kwxUfiNiJCIuAUaB7wx6\nXqe3d9JLA5HXed3PvANLgAXAy9Omvwz8Ypdj6YbHgMuA/wSWAp+j+NKsSCkd6GFcVTuRYmPYLM8n\nzpx9INwH/APwHPALwBeAb0fE2Y0frLUUEQHcCjySUpo432Ng89uivTBA+Y2IFcB3KaqLvUGx1/1M\nRJzNAOa1VXsbLw9MXrs9mA+VlNLWSU93RcTjwI+Biyn+3tGASCl9fdLTsYj4IfAMcD7wcE+CKscm\nYBnwkV4H0iVN2ztg+X0KOB1YBPwWcE9EnNfbkCrVtL0ppe8PUl67fQLcPuAIxckGk50A9Oa2OV2U\nUtoPPA3U9uzQNu2lOBdiKPMMkFJ6jqK/1zbXEXE7cAFwfkppz6SXBjK/s7R3hjrnN6X0Vkrp2cZg\ntoHib/SrGdC8ztLeZvPWNq9dHcxTSoeBHcCqiWmNv7VWAY92M5ZeiIhjKTrJrBuKumt8IfYyNc/H\nUZwtPPB5BoiIk4DF1DTXjYHtIuBXU0ovTH5tEPM7W3tbzF/r/E4zAiwYxLy2MEJxuHeGOue1F3+z\nfxnYHBE7gMeB9cA7gM09iKVSEfEl4FsUf63/HPBHwGFgSy/jKkNEHEPxw2TizrzvjYjTgddSSi9S\nHHe8PiJ+RHHXvD+muGrhH3sQ7rzN1t7G4waKY297G/P9CcW/MFtnLq2/RcQmistzLgQORMTEntr+\nlNLE3Q8HJr9ztbeR+4HIb0TcTHGc+AXgncClwLnATY1ZBiavMHt7BymvQPcvTWucU3ANRUc5SHFi\nwspen9ZfUTu3UHwRDlJ0pruBU3odV0ltO4/i8p0j0x53TJrncxSXurxB8eU4tddxV9FeihNr7qfY\nIBwCngW+Aryr13F32NZm7TwC/M60+QYiv3O1d5DyC/x1I/6DjfY8AHxsEPM6V3sHKa8pJW+BKklS\n3VmbXZKkmnMwlySp5hzMJUmqOQdzSZJqzsFckqSaczCXJKnmHMwlSao5B3NJkmrOwVySpJpzMJck\nqeYczCVJqjkHc0mSau7/APAgGuehUE1gAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "LSTMStateTuple(c=, h=) LSTMStateTuple(c=, h=)\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n", + "ready to train..." + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(RNNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "for i, (grad, var) in enumerate(gradients):\n", + " if grad is not None:\n", + " gradients[i] = (tf.clip_by_value(grad, -10, 10), var)\n", + " \n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model overview...\n", + "\tvariable \"basic_lstm_cell/weights:0\" has 20480 parameters\n", + "\tvariable \"basic_lstm_cell/biases:0\" has 256 parameters\n", + "\tvariable \"W_z:0\" has 3072 parameters\n", + "\tvariable \"W_v:0\" has 384 parameters\n", + "\tvariable \"W_r:0\" has 60 parameters\n", + "\tvariable \"while_loop/basic_lstm_cell/weights:0\" has 20480 parameters\n", + "\tvariable \"while_loop/basic_lstm_cell/biases:0\" has 256 parameters\n", + "total of 44988 parameters\n" + ] + } + ], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no saved model to load.\n" + ] + } + ], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 0/40000\n", + "\tloss: 0.6937\n", + "Iteration 100/40000\n", + "\tloss: 0.6921\n", + "Iteration 200/40000\n", + "\tloss: 0.4698\n", + "Iteration 300/40000\n", + "\tloss: 0.3195\n", + "Iteration 400/40000\n", + "\tloss: 0.2620\n", + "Iteration 500/40000\n", + "\tloss: 0.2429\n", + "Iteration 600/40000\n", + "\tloss: 0.2198\n", + "Iteration 700/40000\n", + "\tloss: 0.2103\n", + "Iteration 800/40000\n", + "\tloss: 0.1918\n", + "Iteration 831/40000" + ] + } + ], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + " global_step = i\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=global_step)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "fig, ((ax1,ax2),(ax3,ax5),(ax4,ax6),) = plt.subplots(nrows=3, ncols=2)\n", + "plt.rcParams['savefig.facecolor'] = \"0.8\"\n", + "fs = 12 # font size\n", + "fig.set_figwidth(10)\n", + "fig.set_figheight(5)\n", + "\n", + "ax1.imshow(_X.T - _y.T, interpolation='none') ; ax1.set_title('input ($X$) and target ($y$)')\n", + "ax2.imshow(_y_hat[0,:,:].T, interpolation='none') ; ax2.set_title('prediction ($\\hat y$)')\n", + "\n", + "ax3.imshow(_w_w[0,:,:].T, interpolation='none') ; ax3.set_title('write weighting ($w_w$)')\n", + "ax4.imshow(_w_r[0,:,:,0].T, interpolation='none') ; ax4.set_title('read weighting ($w_r$)')\n", + "\n", + "ax5.imshow(_f[0,:,:].T, interpolation='none') ; ax5.set_title('free gate ($f$)') ; ax5.set_aspect(3)\n", + "ax6.imshow(_g_a[0,:,:].T, interpolation='none') ; ax6.set_title('allocation gate ($g_a$)') ; ax6.set_aspect(3)\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/repeat-copy/models/checkpoint b/repeat-copy/models/checkpoint deleted file mode 100644 index ac2497e..0000000 --- a/repeat-copy/models/checkpoint +++ /dev/null @@ -1,2 +0,0 @@ -model_checkpoint_path: "model.ckpt-5000" -all_model_checkpoint_paths: "model.ckpt-5000" diff --git a/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 deleted file mode 100644 index 63af8c5..0000000 Binary files a/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-1000.index b/repeat-copy/models/model.ckpt-1000.index deleted file mode 100644 index 9c25af7..0000000 Binary files a/repeat-copy/models/model.ckpt-1000.index and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 deleted file mode 100644 index 577a6ba..0000000 Binary files a/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-2000.index b/repeat-copy/models/model.ckpt-2000.index deleted file mode 100644 index 7f94a9c..0000000 Binary files a/repeat-copy/models/model.ckpt-2000.index and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-2000.meta b/repeat-copy/models/model.ckpt-2000.meta deleted file mode 100644 index 77dbcc0..0000000 Binary files a/repeat-copy/models/model.ckpt-2000.meta and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 deleted file mode 100644 index e423541..0000000 Binary files a/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-3000.index b/repeat-copy/models/model.ckpt-3000.index deleted file mode 100644 index 10b5931..0000000 Binary files a/repeat-copy/models/model.ckpt-3000.index and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-3000.meta b/repeat-copy/models/model.ckpt-3000.meta deleted file mode 100644 index ea8a258..0000000 Binary files a/repeat-copy/models/model.ckpt-3000.meta and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 deleted file mode 100644 index bb7e770..0000000 Binary files a/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-4000.index b/repeat-copy/models/model.ckpt-4000.index deleted file mode 100644 index 32b887c..0000000 Binary files a/repeat-copy/models/model.ckpt-4000.index and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-4000.meta b/repeat-copy/models/model.ckpt-4000.meta deleted file mode 100644 index ea8a258..0000000 Binary files a/repeat-copy/models/model.ckpt-4000.meta and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 deleted file mode 100644 index 0ee35fb..0000000 Binary files a/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 and /dev/null differ diff --git a/repeat-copy/models/model.ckpt-5000.index b/repeat-copy/models/model.ckpt-5000.index deleted file mode 100644 index b0059a3..0000000 Binary files a/repeat-copy/models/model.ckpt-5000.index and /dev/null differ diff --git a/repeat-copy/nn_controller.py b/repeat-copy/nn_controller.py index f2c8111..144c977 100755 --- a/repeat-copy/nn_controller.py +++ b/repeat-copy/nn_controller.py @@ -28,5 +28,5 @@ class NNController(Controller): h2 = tf.nn.elu(z2) return h2, state - def zero_state(self): + def get_state(self): return LSTMStateTuple(tf.zeros(1), tf.zeros(1)) diff --git a/repeat-copy/nn_models/checkpoint b/repeat-copy/nn_models/checkpoint new file mode 100644 index 0000000..3e7aff5 --- /dev/null +++ b/repeat-copy/nn_models/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "model.ckpt-10000" +all_model_checkpoint_paths: "model.ckpt-10000" diff --git a/repeat-copy/nn_models/model.ckpt-10000.data-00000-of-00001 b/repeat-copy/nn_models/model.ckpt-10000.data-00000-of-00001 new file mode 100644 index 0000000..44274e0 Binary files /dev/null and b/repeat-copy/nn_models/model.ckpt-10000.data-00000-of-00001 differ diff --git a/repeat-copy/nn_models/model.ckpt-10000.index b/repeat-copy/nn_models/model.ckpt-10000.index new file mode 100644 index 0000000..ca2dfba Binary files /dev/null and b/repeat-copy/nn_models/model.ckpt-10000.index differ diff --git a/repeat-copy/models/model.ckpt-5000.meta b/repeat-copy/nn_models/model.ckpt-10000.meta similarity index 99% rename from repeat-copy/models/model.ckpt-5000.meta rename to repeat-copy/nn_models/model.ckpt-10000.meta index ea8a258..992032c 100644 Binary files a/repeat-copy/models/model.ckpt-5000.meta and b/repeat-copy/nn_models/model.ckpt-10000.meta differ diff --git a/repeat-copy/repeat-copy-nn.ipynb b/repeat-copy/repeat-copy-nn.ipynb new file mode 100644 index 0000000..73dc39f --- /dev/null +++ b/repeat-copy/repeat-copy-nn.ipynb @@ -0,0 +1,453 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Differentiable Neural Computer (DNC) using a NN Controller\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from nn_controller import NNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 10000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"nn_models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAD2lJREFUeJzt3X+MHPV5x/H3cxdzTkxsq3ZqWyUNpCRFvotobIeUJgYa\n23JEVdNKlYOLVEESIUPSP65qS5CRnFKHpI2IaaGO+ss4FeHcVE1URw0Y29CmiFBXdkLjMxSFH4GW\ns4tBOJKxHZ/v6R+zR+/X7s0zO7O7M/t5SfvHzs7MPt/5fme+Ozszz9fcHRERESmvnnYHICIiIs1R\nZy4iIlJy6sxFRERKTp25iIhIyakzFxERKTl15iIiIiWnzlxERKTk1JmLiIiUnDpzERGRkntbkSs3\ns0XAeuBF4EyR3yUiIlIxc4GLgb3u/lqjGTN15mb2GeD3gaXAU8Dvuvt/zDDreuDrWb5DREREALgB\neLDRDOHO3Mw+AdwN3AwcBAaBvWb2fnc/MWX2FwGue+DXWXzZokkf7Bs8wLrta6atv2fVrlA8Y4du\nCs3fLvsG97Nu+9qm1xNNpb9zVayKP3VoNPYFdeRV3pDgxsmrrdUtazCe3mA80VEVovtKvfDrlbfo\nthZv+3Mia68zfS/JOclkee0nbVNnY+q4nE1xbfME8E2o9aWNZDkzHwT+0t3/DsDMNgO/BnwS+NMp\n854BWHzZIpatWDrpg76FfdOmQfwi/tgM6+hEfQtmLm9UfFycyAENlq04F/2CGeVV3pBoZx5cfb22\nVres0c48GE+4Mw/WR73w59bZd4tua/G2f0Fk7XWmzwWWTZua137SNnU2po7L2RTbNoEUl6lDdWRm\nc4CVwIHxaZ4Mu7YfuDIanYiIiDQv+oNrMckJxPEp04+TXD8XERGRFtOjaSIiIiUXvWZ+AjgPLJky\nfQlwrN5C+wYP0Lewb9K0BT8/P/jV5da/aXm7Q2ipbipvN5UVoP/6birvQLsDaKnuqttO23d/CByZ\nMi39E92hztzdz5nZIWANsAfAzKz2/s/rLbdu+5rW3wzVYQY29bc7hJbqpvJ2U1kB+ruqvN3VmQ90\nVOdWvM7adz9Qe000AvxVqqWz3M3+FWBXrVMffzTtHcCuDOsSERGRJoU7c3f/hpktBu4k+Xv9B8B6\nd3817+BERERkdpkywLn7DmBHzrGIiIhIBrqbXUREpOSypHNdDfwBSfKYZcBvuPueWRZKXimMnf9c\nKJ4e+2Jo/rB0Yb9lbOz2YuKoSbkZ37LlfCxT1TaLZfEKC8R/x1iHZdmKZpjr/VJo/vNjsbZfdtGs\nWV/ojbXNLWM/jX1BRLQt9MTaQuHHneCBZKzgthluCz3B41Rwe0aPPeHjcsq2OXL4HDtXpVtnljPz\neSTXyW8lnlFSREREcpblBriHgYfhrcfSREREpI10zVxERKTk1JmLiIiUnDpzERGRksv0nHnUvsH9\n9C2YnJu9f9PyDkulJyIi0h7DQ8MM7z46adqZN86mXr4lnfm67Wu7Pje7iIhIPf2b+qeNezBy+Bg7\nV92favksz5nPAy7l/5/ce6+ZXQ687u4vR9cnIiIizclyZr4KeIzkGXMH7q5N/xrwyZziEhERkZSy\nPGf+r+jGORERkY6hTllERKTkQmfmZnY78JvAZcBp4AngNnd/NreICs4pF8393hvMr93TE8sVX3Qu\n96K3ZzT3+xd6L0g977Zg/uVoPuVwvuloPuvR4DgDwbbWceMAFJ3PejTa1tK3n/A4AAUnso4ep8IK\nHmeg8ONaUPQ4FVVcbvn0XXT0zHw1cC/wYWAtMAd4xMzeHlyPiIiI5CR0Zu7u1058b2Y3Av9LMoLa\n4/mFJSIiImk1e818IckfTq/nEIuIiIhkkLkzr42Ydg/wuLsfnW1+ERERKUYzGeB2AMuBj+QUi4iI\niGSQqTM3s/uAa4HV7j4y2/zKzS4iItLIkdprojOpl86SzvU+4Drgand/Kc0yys0uIiLSyEDtNdEI\n8Neplo4+Z74D2ARsAE6Z2ZLaRyfdPf1PCBEREclN9Aa4zcB84F+AVya8NuYbloiIiKQVfc5c6V9F\nREQ6jDpnERGRkoteM98M3AJcXJs0DNzp7g83XNA9dfLanp5YDuCwnlhC6PPBHMnRHMZRxeUAzsaC\nPwe3nP9p6nkjubWBeL7pottaUDSNftHjAHRaW4tLv0ULHwcgehyJts1g44m2haJzxYfz9EfHhIi2\nzWA80faQdhyDkcOj7FyVbp3RM/OXgduAFSQpXB8F9pjZ8uB6REREJCfRa+b/PGXSHWZ2C8nAK8oC\nJyIi0gaZM8CZWQ/JXex9wL/lFpGIiIiEZEkaMwB8D5gLvAlsdPcf5R2YiIiIpJPlbvZngMuBK4D7\ngN1m9sFcoxIREZHUwmfm7j4KPF97+30zu4LkDveb6y2zb/AAfQun5Ga/fjkDm3TfnIiIyPDQMMO7\nJ996duaNs6mXb2bUtHE9QG+jGdZtX6Pc7CIiInX0b+qnf8rgYyOHj7Fz1f2plo8+Z34X8BDwEvBO\n4AbgKmBbZD0iIiKSn+iZ+c8CXwOWASeB/wTWu/tjeQcmIiIi6USfM/90UYGIiIhINsrNLiIiUnJN\n3QBnZp8D7gLucfffyyekmHDOY4vls+420ZzH26yD8nEHEzyH204w7350/b3RvP7B3OlRhefLDube\nj6//gvQzR7dldByAgsdsGBuN5n4vNq9/VHgcgOi4DUFbRoPHwYJy+0f2wcxn5mb2IZLH0Z7Kug4R\nERFpXqbO3MwuBB4APg28kWtEIiIiEpL1zPwvgG+7+6N5BiMiIiJxWXKzXw/8EpBylFUREREpUjRp\nzEXAPcBad4/dISAiIiKFiJ6ZrwTeBRw2e+s+u17gKjP7LNDnPv2+ROVmFxERqe/I0DDDQ5Nzs589\nWVxu9v3AB6ZM2wU8DXxppo4clJtdRESkkYFN/QzMkJv9b1cWkJvd3U8Bk346mNkp4DV3fzqyLhER\nEclHHhngCk5dISIiIo00PQSqu38sj0BEREQkG6tzmTuflZutAA596tBN6a+ZR+OJ5pwssLzJ+mOz\n97wtmCI0mFax7JszIhp7WKdtzA5ra1Gdtjkjwm2t6B2lkzYOGVIfF3xciyp6c6Zd/4Rr5ivd/XCj\neUN/s5vZVjMbm/I6OvuSIiIiUpQsf7MfAdYA478tRvMLR0RERKKydOaj7v5q7pGIiIhIJlnuZn+f\nmf2PmT1nZg+Y2btzj0pERERSi3bmTwI3AuuBzcAlwHfNbF7OcYmIiEhK0aQxeye8PWJmB4EfAxuB\numlq9g3up2/B5HSuC94zn4/fuz7y9aV2ZOhoV6WvHR4apn9KNqOq6ra6PTI0PC1TVVV1UzuG7mvL\nnVS/rU7nOom7nzSzZ4FLG823bvvaaY+m/f2Gf2jmq0tneHeX7SS7j3bMTlK0rqvboaPd05l3UTuG\nLmzLHVS/zaZzbSoDnJldSNKRjzSzHhEREcku+pz5l83sKjN7j5n9CvAt4BwwVEh0IiIiMqvo3+wX\nAQ8Ci4BXgceBX3b31/IOTERERNKJ3gC3Kbj+uQAnnj4x7YOzJ88ycvjYTF8S+4YOS2NYL8VmvfL2\nBMMZm2mbNQqnTZvzzBt16rdA7UrnerZeWTstnWtOba3uvhtUhl29XjuuajrXum05qNOOa/XkVb9F\nteUJfefcWddZcG723wa+XtgXiIiIVN8N7v5goxmK7swXkTyT/iJwprAvEhERqZ65wMXA3tkuZxfa\nmYuIiEjxmno0TURERNpPnbmIiEjJqTMXEREpOXXmIiIiJdeWztzMPmNmL5jZaTN70sw+1I44imZm\nW81sbMrr6OxLdj4zW21me2rD4Y6Z2YYZ5rnTzF4xszfNbJ+ZNczh38lmK6+Z3T9DXX+nXfE2w8xu\nN7ODZvYTMztuZt8ys/fPMF8l6jdNeatSv2a22cyeMrOTtdcTZvbxKfNUol5h9vJWpV6hDZ25mX0C\nuBvYCnwQeArYa2aLWx1LixwBlgBLa6+Ptjec3MwDfgDcygzpS8zsNuCzwM3AFcApknq+oJVB5qhh\neWseYnJdR5MsdYrVwL3Ah4G1wBzgETN7+/gMFavfWctbU4X6fRm4DVgBrAQeBfaY2XKoXL3CLOWt\nqUK9gru39EUyJvqfTXhvwH8Df9jqWFpQ1q3A4XbH0YJyjgEbpkx7BRic8H4+cBrY2O54Cyrv/cA3\n2x1bQeVdXCvzR7ukfmcqb5Xr9zXgpqrXa53yVqZeW3pmbmZzSH4dHRif5skW3Q9c2cpYWuh9tb9m\nnzOzB8zs3e0OqGhmdgnJL9yJ9fwT4N+pbj0DXFP7m/YZM9thZj/T7oByspDk34jXoSvqd1J5J6hU\n/ZpZj5ldD/QB3616vU4t74SPKlGvTY1nnsFioBc4PmX6ceAXWxxLKzwJ3Aj8F7AM+DzJTjPg7qfa\nGFfRlpIcDGeq56XTZ6+Eh4B/BF4AfgH4IvAdM7uy9oO1lMzMgHuAx919/H6PytZvnfJCherXzAaA\n75FkF3uT5Kz7OTO7kgrWa73y1j6uTL22ujPvKu6+d8LbI2Z2EPgxsJHk7x2pCHf/xoS3w2b2Q+A5\n4BrgsbYElY8dwHLgI+0OpEVmLG/F6vcZ4HJgAfBbwG4zu7q9IRVqxvK6+/erVK+tvgHuBHCe5GaD\niZYArR1iqw3c/STwLFDau0NTOkZyL0RX1jOAu79A0t5LW9dmdh9wLXCNu49M+KiS9dugvNOUuX7d\nfdTdn691ZltI/ka/hYrWa4PyzjRvaeu1pZ25u58DDgFrxqfV/tZaAzzRyljawcwuJGkkDQ8UZVfb\nIY4xuZ7nk9wtXPl6BjCzi4BFlLSuax3bdcCvuvtLEz+rYv02Km+d+Utdv1P0AL1VrNc6ekgu905T\n5nptx9/sXwF2mdkh4CAwCLwD2NWGWAplZl8Gvk3y1/rPAX8EnAOG2hlXHsxsHskPk/GRed9rZpcD\nr7v7yyTXHe8wsx+RjJr3xyRPLfxTG8JtWqPy1l5bSa69HavN9yck/8Lsnb62zmZmO0gez9kAnDKz\n8TO1k+4+PvphZep3tvLW6r4S9Wtmd5FcJ34JeCdwA3AVsK02S2XqFRqXt0r1CrT+0bTaPQW3kjSU\n0yQ3Jqxq9239BZVziGRHOE3SmB4ELml3XDmV7WqSx3fOT3ntnDDP50kedXmTZOe4tN1xF1Fekhtr\nHiY5IJwBnge+Cryr3XFnLOtM5TwP/M6U+SpRv7OVt0r1C/xNLf7TtfI8AnysivU6W3mrVK/uriFQ\nRUREyk652UVEREpOnbmIiEjJqTMXEREpOXXmIiIiJafOXEREpOTUmYuIiJScOnMREZGSU2cuIiJS\ncurMRURESk6duYiISMmpMxcRESk5deYiIiIl93+wmFeJOTGfowAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n", + "ready to train..." + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(NNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "for i, (grad, var) in enumerate(gradients):\n", + " if grad is not None:\n", + " gradients[i] = (tf.clip_by_value(grad, -10, 10), var)\n", + " \n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model overview...\n", + "\tvariable \"W1:0\" has 2048 parameters\n", + "\tvariable \"b1:0\" has 128 parameters\n", + "\tvariable \"W2:0\" has 32768 parameters\n", + "\tvariable \"b2:0\" has 256 parameters\n", + "\tvariable \"W_z:0\" has 12288 parameters\n", + "\tvariable \"W_v:0\" has 1536 parameters\n", + "\tvariable \"W_r:0\" has 60 parameters\n", + "total of 49084 parameters\n" + ] + } + ], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loaded model: nn_models/model.ckpt-7000\n" + ] + } + ], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 7100/10000\n", + "\tloss: 0.0162\n", + "Iteration 7200/10000\n", + "\tloss: 0.0112\n", + "Iteration 7300/10000\n", + "\tloss: 0.0105\n", + "Iteration 7400/10000\n", + "\tloss: 0.0176\n", + "Iteration 7500/10000\n", + "\tloss: 0.0053\n", + "Iteration 7600/10000\n", + "\tloss: 0.0180\n", + "Iteration 7700/10000\n", + "\tloss: 0.0144\n", + "Iteration 7800/10000\n", + "\tloss: 0.0159\n", + "Iteration 7900/10000\n", + "\tloss: 0.6691\n", + "Iteration 8000/10000\n", + "\tloss: 0.3837\n", + "\n", + "\tSAVING MODEL\n", + "Iteration 8100/10000\n", + "\tloss: 0.0386\n", + "Iteration 8200/10000\n", + "\tloss: 0.0129\n", + "Iteration 8300/10000\n", + "\tloss: 0.0166\n", + "Iteration 8400/10000\n", + "\tloss: 0.0266\n", + "Iteration 8500/10000\n", + "\tloss: 0.0160\n", + "Iteration 8600/10000\n", + "\tloss: 0.0153\n", + "Iteration 8700/10000\n", + "\tloss: 0.1232\n", + "Iteration 8800/10000\n", + "\tloss: 0.1623\n", + "Iteration 8900/10000\n", + "\tloss: 0.0657\n", + "Iteration 9000/10000\n", + "\tloss: 0.0635\n", + "\n", + "\tSAVING MODEL\n", + "Iteration 9100/10000\n", + "\tloss: 0.3011\n", + "Iteration 9200/10000\n", + "\tloss: 0.1336\n", + "Iteration 9300/10000\n", + "\tloss: 0.1150\n", + "Iteration 9400/10000\n", + "\tloss: 0.0853\n", + "Iteration 9500/10000\n", + "\tloss: 0.0779\n", + "Iteration 9600/10000\n", + "\tloss: 0.0704\n", + "Iteration 9700/10000\n", + "\tloss: 0.0385\n", + "Iteration 9800/10000\n", + "\tloss: 0.0153\n", + "Iteration 9900/10000\n", + "\tloss: 0.0030\n", + "Iteration 10000/10000\n", + "\tloss: 0.0191\n", + "\n", + "\tSAVING MODEL\n" + ] + } + ], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + " global_step = i\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=global_step)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA94AAAHHCAYAAABJK4BRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xm8HFWZ//HPk5uQhCUJOwhocEAgCdtN1CGsIyDKzIjb\nYK6IggvihpP5jQiyBKICI6NRZBhxgaBAGBwBoyMgq6BBEAJKgIgsYQ8QQhay597n98epm/TtW327\nTndXb/f7fr36ldzqU6eeqlPdp09VnXPM3RERERERERGRfAxpdAAiIiIiIiIi7UwNbxEREREREZEc\nqeEtIiIiIiIikiM1vEVERERERERypIa3iIiIiIiISI7U8BYRERERERHJkRreIiIiIiIiIjlSw1tE\nREREREQkR2p4i4iIiIiIiORIDW8RERERERGRHKnhLSIiIiIiIpIjNbylLszsBDPrMbM3NzqWapjZ\nqWb2aA3y+ayZPWNmw2oRV62Y2Tlm1tPoOJpFJeXdrGUrIiKNl1bP1vo3UjP85hqo/jSzocWxqe6U\nwUANb6kXT15NwcwOMLNpZjYqYp0tgFOBC2oQwkxgE+CzNcirljKVUyXHr5HqXN4zac6yFRGRxkur\nZ6N/I5Wp1xr6m2ug+tPMDPghcJmZ7VTw1kxUd0qbU8Nb6uWnwEh3f7bRgSQmA2cDYyLW+RTQAVxT\n/IaZHWdmc5IrzGvNbGbBe/+dLF9sZjeY2bbuvga4Avi36najYSo5fo1U0/IeSBuUrYiI1Fclv5EG\nqtca/ZtroPrzPwjxHQucb2ZbgupOGRzU8Ja68GBto+MoYBWscwIwO20/3P0qd58M/Bl41N1PKHj7\nImA28BZ3f7+7v5osvxYYa2aHVRBLo1Vy/MpnarZpHvlS4/LOoJXLVkREiuRYP1X6G6lkvdYEv7lO\noHT9+QN3v9PdFwNfIDTQe6nulLamhrfURXF/o94+Tmb2d2Y208xeN7MlZnaZmY0oWrc37R5mdq2Z\nLTWzRWb2XTMbXpBuppk9nbLtPv2pzGwa8K3kzwVJ3t0D9YUys7HAPsCtZXb1R8DeZjYxWW9b4BTg\nQ+6+vDChu88FFgPHlMkTM3uzmV1iZvPNbGWy/9ea2VvS9jXjcT3IzP5kZqvM7G9mdlK5OJL1Bjx+\nFcS6l5ldbWaLgbsL3j/MzO4vjK9E37g3Jfu30MxWm9k8Mzsxa7wl9nEs2co7VUzZiohIfUT8nihX\nPw1Y7xSky1TPFv9GKtjGT8zshWQbTyV169AM9XBqH28z29/Mbkz2e7mZ3Wpm7yxxjMr+jiixL2MZ\noP5096cK/r/c3RcV/K26U9ra0EYHIINGcX+j3v9fCzwFnAZ0Ap8GXgZOL5H26STt3xMatGMIV1bT\ntlFq278A3gZMAb4MvJYsf5XSJid5zB0gDcBVwIXAp83sMeA8YKq7d5dIPxc4sEyeAG8n7PMs4Hlg\nLPB54A4zG+fuq5N0mY6rmU0AbgZeITyqNgw4J/m7nHLHLzbWnwOPJ7FZEt/+wI3Ai8BZhO+qs4BF\nBethZtsB9wLdhCcLFgHvBX5iZlu4+0XAdWXiTZO1vAeStWxFRKQ+Yn5PQHr9lKXeia1n+/xOMbMd\ngT8Bo4BLgb8COwEfBjalfL3W7/eQmY0D7gKWEvperyf0p77TzA5x9z+lHKNyv8/SVFt/qu6U9uXu\neumV+wv4BKGSenPy9zSgB/hhUbpfAK8ULetNe13R8ouTPCckf18OPJWy7WlAd9Gy/1cYT4b4pyfp\nN82QdiahYrsU2KFM2h8Ab2TIc3jKsnckx+W4lGM14HEFrgdWADsVLNsDWFd8rErEU/L4VRDrz1LS\nzwaWA9sXLHsrsLYwPuDHhMb9mKL1ryZcNR9e6/IGPka4uPLdgmWbEH6c7RhbtnrppZdeetXnFfF7\nYqD6KWu9k7mepf9vpCuSdPsPsC8D1cOfKH4viWcVodtb77Idkt8rd6Qco7K/z0rElVp/qu7USy/X\no+bSUE5onBa6G9jazDZPSftfRcu+T7gCfXQ+4fWxNbDe3VdmSPsjYAtgnrsvLJP2dWBkuce3PAw6\nAmyYhmMrwpXoJYQr0X2SM8BxNbMhwLuB6939hYJt/JVwdb4q1caaxHc4cIO7v1yQ71OEu+CFPgj8\nCugws617X8BvgdEp28sqtbzNbA9C4/864JMFb00EtgMKyztT2YqISF1l/T2RVpfCwPXOGKCzmnrW\nzIzwqPVsd38was9K5zkEODKJ55mCeBYSLhgcVPS7K+b3WbF+9afqTpFADW9ptOIRN19P/t0yJe0T\nRX8/SbgqO7bGMVVrIuFqeFeGtL2Doww47YeZjTCz6Wb2LLCG8GjbK4TG5eiUVQY6rtsCI+l/PCE8\nzlaVCmIt7pe/3QDxbVhmof/8GOAkwuN1ha/LCvKqpQOBG4DDgPsLlk8G5rh7YTlmKlsREam7rL8n\n+tRPGeodJ9Q71dSz2xIeMX+kTLoY2xIeUX885b3HCO2BXYqWx/w+K0d1pwjq4y2NV6rvc5ZRqNPm\nwUzTUWJ5jNeAoWa2mbuvKJXIzI4lXLn9T2CGmY1394Eqzy2BlYV3iUu4mPDo2Azgj4RHwxz4H9Iv\noFVzXKsVG+uqCrfTm9eVhMfy0vylwrxTy9vdLwMws/cTujb0mkzoO1coa9mKiEhjlfr9UFw/Za13\navG7o9Eq/R3Rr/5U3SkSqOEtrWR34JmCv3cjVIK9V6RfJ30+y7Epy2KvpM5P/t0VmJeWwMwOJfTp\nvsjMxhAGLzmJMPBJKbsSrjaX8yFgprufWrC94VQ2j/arhB8Tu6e8t2fGPAY6ftXG+gqwmlC+xQpj\nfpXQD7zD3W8vk2fNytvCnKMTCYPc9DoA+HZRHlnLVkRE6qvc74lSMtU7yaPdldazrwLLgAll0sXU\na68CKwl9zIvtRbjb/1xEfgNJrT9Vd4roUXNpHUaY77HQKYSK56bk7yeB0clIomGlMDLo+1Py672L\nmbUxeE8Sw6TU4MJooYd7Mpqpuy8hDETyscIpSlJ0AnMybL+b/p/XU6jgqrq79xD6mL3fzHbuXW5m\nexH6pGUx0PGrKtYkvluT+HYoiG834D1F6X4BfMjMxhfnY2bbZIw3zUDl/XfAEnd/LtnOWwmP0P+p\nKF3WshURkfoZ6PdE8TgifWStd6qpZ5PHrm8A/tnMBhqnJHO9lsTzW+CYoinLtid0i7vb3d8ol09G\npepP1Z0y6OmOt7SSXc3sl4SG9mTgOOBKd384ef8a4D+AG8zsImAz4GRCf6riyusBQsVwnpldQxg9\ndLa7pz727O5Pm9k84AjCqOUbmNk7gNPd/QNFq/04ifEjwE+L87Qw1/dWhAq2nF8Dx5vZMuBRwlXi\nwwn9pysxjdCI/b2ZXUKY5uSLhKvT+2RYf6DjV4tYzyH8OJljZv9N+K76QhLfvgXpTiP0GbvXzH6U\nbG8rwlX1dwG9je+alTfhbscwM7PkB9LngPvcfV1vgsiyFRGR+ir1eyL1ibYiWeudaurZrxEGQ7vL\nzH5IuAP8JsLd4gPdfRmR9RpwJqFO+0MSTzfhqbxNgFNLrBNtgPpTdacMerrjLa3CCQ3YNcD5hDkz\nLyLMKxkSuC8m3N1eQWiAH0+oIH/dLzP3+wmV0D6E/kZXEwYfGchlhCvQwyH0VTKzu4A/AIeY2Yar\n2Gb294RRUh24yMx+kYzuXehfgGfc/c4M+38KofH+UUL/8e0JldobVDAASXKx4t2Ex7rPJcxdejYZ\nK7syx+/L1cbq7nMJP1gWE6Ym+SShMX4b4TH03nSvEKYquwz4AOGY987HempBuqrLuyCvvxLOve+Y\n2WmEEW7vKFo3pmxFRKR+yv6eGHDl7PVOxfWsu78IvJMwj/hHge8RpuO6nfDIeHS95u6PAgcDDxN+\nG51FeLT+sCSvWupXf6ruFAHrO5CgSPMxs2mEymrbpHHdqDhGER5nP9XdLy+XvkxemwALgPPc/eIa\nhDcomNn1wDh3T+unVutt9StvMxsGfAf4nrs/kXRl+Buwt7s/naRR2YqINKFm+T3R7orrT9WdIoHu\neItklDzadSHwlRpkdyJhTsu0OUKFMC1Z0d+7E+ZYLb5CnosS5b0f8HHCtCwQ5mw/o/eHQ0JlKyIi\ng1ZK/am6UwTd8ZYWoCvUg5OZvUjoH/YUYWT6kwl95Drd/ckGxTSU8IjeWmBH4I/u/j+NiEVEROLo\n90RjqO4UCTS4mog0qxuBKcAOhL54c4CvNarRDeDu64FvNGr7IiIirUZ1p0igO94iIiIiIiIiOcr1\njreZbQ0cRRgsYfXAqUVERNrSCEJ3iZvd/bUGx1IzquNFRESy1/F5P2p+FHBVztsQERFpBccRpvxp\nF6rjRUREgrJ1fEUNbzP7AvDvhL6Xfwa+5O5/Skm6AOCYK/+Zbfbaps8bt0y9lSNnHFHJ5usn4jH8\nIZNmpi6/mfDLpFjPAydWFFKzSivP2F4Ml02KOx0/df+6zGlLlU8pPfefkLr8lqm3ceSMw/ss64jM\nO0ZsR5CY86pU+ZT6bMaWzyfvX5857WWThkXlHafUUUz/dH7qgexxx57kMedKdNlHnLMQ95nI8/sq\n/nui1LlyE2Hq9z65R+Wd3zm7CLgOkjqx2cXW8Vd+Hvbcqe8bU38GM47vv8KKIyN/dkScIJvP647L\nO8Y26YunfgNmnNl/uS/KnvWCfXeICmXXvy7Mnjh7NRmsTV889WKY8cW+y3yruKyXHhuXfvTXs6f1\nA+LyXrF5+nn4tf/XzXnf7uizbM2Q4alpS9n65RWZ064dFZU1mzyePe1D+6bPuvn9qc/zpRk791s+\nYtJfo2LZ65fZ0/oXorJmxcvZ024+On351GUwI+34RsTiJfIuZdGhm2dOu97jvgt3+NOS1OVTL4EZ\nn++/fPW+2fMeMS8qFF7ozP7h33le5BiG09MXT10IM4q/KnvisuZfI9Kemz3pY+vhY8uBDHV8dMPb\nzD4CfBs4CbgPmArcbGZvc+9X1awG2Gavbdixs+/RGj56eL9lTSem4V1i+QjC8I3Fepp93yOllWf8\n8AFxja8dO0v8QkgRO29eqfIZPqb/fnakpqyN6MZXxHlVqnxGpOxjEFs+Mb/4NonKO06po5j+6YyK\nO7bhHZG2VmWfds5C3Gciz++r+O+JUudKWnnGZV6Hc7bpH8eupI7fcyfo3LXvG2M27b8MYHmnxQUU\nUYSjYhuZMdIqcmD0KOic0H+5v5Q96806486lPWM+vGuisi6ZfvTm0Pm2vst8u7isF0cW/VYp508p\nHtHIAFg+Oj2YUaNhv6JzdNWQuFp+++ezp12zZVTWDI84hqs6N01dvvmYDvZIeW9kXCip530pPqJ8\nmkLLI87xUSU+PqMNUj9aY7PnHXtxaWFn9nNlbWTD+81L05ePSflsAqzcP/vJsunauLpyy87svwff\nGvu9XOJEHN0BncXvxV5r3S0ibWX3g8rW8ZXM4z0VuNTdf+ru8wlT/KwEPllBXiIiItI8VMeLiIjk\nIKrhbWbDgInAbb3LPAyLfisQ+aCPiIiINAvV8SIiIvmJveO9DeFpyeLeFy8T+oKJiIhIa1IdLyIi\nkpO8RzUHwmBNw0f3HZxi9FsiR5RoURFdYFra+K5xjQ6hLsZPaf/9HAz7GAyOT6fKs94eBopHqmn6\nrt1Vmfqz0Ke70JtLDEbWbrr+qdER1EdX//EZ29KHp1TSA7O1HDElsmN5i+qK7bTeoqa8q9ER1EdX\n5EB3eZm1OrwKLY3oIh/b8F5E6Mq+fdHy7YGSw2weOeOI5h9ILSfN8lMwbxO6xjc6hLqYMAguMIwf\nJGU5WD6dg+GcDfZudACJvekfy0vADxsQS7SK6vgZx6cPpDYYdL2v0RHUhxre7eOIrsgRw1pUV/rY\ncm2nSw3vuuoaEV6F5q6Dia9nWz/qG8bd1wEPABu+gs3Mkr/nxOQlIiIizUN1vIiISH4qedT8O8BM\nM3uAjVONbArMrGFcIiIiUn+q40VERHIQ3fB292vNbBvCFOfbAw8BR7n7q7UOTkREROpHdbyIiEg+\nKhpczd0vAS6pcSwiIiLSYKrjRUREai+64W1mBwNfIcz1uSPwfnefPeBK7uGVwZAhF8SGlB/LnrSn\n57Tcwsh46Db45pBh+QQSK+L4AZzRsy6fOIgOJeo8jCweerqznytDOpro8xB5FOPOw7ijeEZ39nPl\nmx2Rn4eID1xTfV81kZjvrKb5vhKgwjo+wpohm9Qqq/42XZ9f3svyy/p1xkSlv6jz2cxph0VWfp87\nJHtaWxmX91b3xKXniexJl2w5vHyiApsvW5M57apRkcNjR0xi0JHjKbuGuGPycGT+ne/JnvaqJ+Py\nPu7piMST4/Ime9HDiPJJCg0le4EOjwqEMNFjhO6hMSvEnYiPs0fmtLt+vXh2yoH97pHsaQ/bJSpr\neCUi7Z4RaZcDeQyultiM8OjZ54lvc4iIiEjzUh0vIiKSg0r6eN8E3AQbRjsVERGRNqA6XkREJB/t\nP2GhiIiIiIiISAOp4S0iIiIiIiKSo4pGNY91y9TbGD6m7yAP46eMY0LXuHpsXkREpE4eBuYVLYsY\nbakFTf0ZjNm077Ipk6ErdtAjERGRJjbrlfAqtDRibLq6NLyPnHE4O3buUI9NiYiINNDeyavQS8AP\nGxBLfcw4Hjp3bXQUIiIi+eraLrwKzV0OEx/Mtr4eNRcRERERERHJUSXzeG8G7MbGSX3famb7Aovd\n/blaBiciIiL1ozpeREQkH5U8aj4JuIMwv6cD306WXwF8skZxiYiISP2pjhcREclBJfN4/w49oi4i\nItJ2VMeLiIjkoy6Dq2EWXhn09JyWczARPHvSIR0XRGXd03N65rQZD90GZ/Ssi1shLxHHD+CbHcOi\n0p8ZsZ/d3XHnVUdEeUbu5sYHOBss+rzqXhuV/psdm0SkjjyKeR7DiAPTk+N5FS2yQPP8ro0J5Yzu\nuO+ruPMqTlTcPdk/Dy/NXcdlkyoIqFV0kPnXxBDrjsraPOK7Ic/vhdgv+uHlk2xMGvfd+qW/RMYS\nI2J0XkbFZW0fiEvvZ2VPO3Llmqi81w7P8dpS3CkeJ+K8GsXyqKxPnhMZy0cjkm4Zl7Utjki8VVze\ndESkjWwldUdlHikylhFvRHyYR8TlvR2vlE+UsKlxeR+acZAyALaOyzvqGH4sIu0zQB6Dq5nZ6WZ2\nn5ktM7OXzex6M3tbTB4iIiLSfFTHi4iI5Cf2kt/BwPeBdwJHAMOA35rZyFoHJiIiInWlOl5ERCQn\nUQ8uuPvRhX+b2QnAK8BE4Pe1C0tERETqSXW8iIhIfqrt5DKG0PMppjeGiIiIND/V8SIiIjVSccPb\nzAz4LvB7d3+0diGJiIhII6mOFxERqa1qRjW/BBgHHFgu4S1Tb2X46L5DMY7vGseErvFVbF5ERKS5\nPDLrER65pm87dfWSuBGXm0TmOn7qFTBm077LphwIXQflE5iIiEgjzLovvAotXZV9/Yoa3mZ2MXA0\ncLC7v1Qu/ZEzjmDHzh0q2ZSIiEjLGN81nvFFF5VfmruQyyZd3qCI4sXW8TM+AZ1vzT8uERGRRup6\nR3gVmvsMTPxmtvWjG95JhXwMcKi7Pxu7voiIiDQn1fEiIiL5iGp4m9klQBfwPmCFmW2fvLXU3VfX\nOjgRERGpD9XxIiIi+YkdXO1kYBRwJ/BiwevY2oYlIiIidaY6XkREJCex83hXO/2YiIiINCHV8SIi\nIvmJfdT8ZOBzwNhk0SPAdHe/acAV3cMrgyFDLogJCSx70p6e0/PLu/u0uLwjZDx0G3xzyLDsiSP2\nEeDMnnW55X1Gd0TeEHVghgyNO6+6I8pzSEfsOZv9wPT0NNF5NTTivALO6F6bPe+OuLwjDiFnxJyz\nsSLP8ZhDHvudMmTI+VHpo78P8xJ5DGOOYux3yjcivjtjvgtjztdGqriO7wbWZ9tGDx0Vx1fW+hxH\nj48Ne0X2pGvZJCrrx/bJnnZx5Ll30F8jEo+Iy5sbItPfmz3puuFxQxYNW5PxhK1ExLnS0R2Zd8R5\ntZwtorJ+rOz8BX3tdX/2tN3visvb3hyR9vtxebMgIm2Op0ne1o3I/uEftiLuB2FHzIHZKSprLOIz\n4ZfG5b3+qOxph10fkXHE4Yu9uv0c8FWgE5gI3A7MNrNxkfmIiIhIc1EdLyIikpPYR83/r2jRmWb2\nOeCdwKMpq4iIiEgLUB0vIiKSn4rm8QYwsyGEAVeGA3fXLCIRERFpKNXxIiIitVXJPN4TgHsIvXtW\nAse6+xO1DkxERETqS3W8iIhIPioZwXQ+sC/wDuBi4Boz27+mUYmIiEgjqI4XERHJQfQdb3dfDzyV\n/Pmgmb2DMArqSaXWuWXqbQwfM7zPsvFTxjGhS+O1iIhI+5g36xEemdW3O/SapTmOtl1jldTxU38G\nYzbtu2zKZOianFeUIiIi9TfrNph1e99lS9/Ivn7FfbwLDKHM5AlHzjicHTt3qMGmREREmteErvFM\n6BrfZ9lLcxfyk4mXNyiiqpWt42ccD5271ikaERGRBuk6PLwKzX0cJn422/qx83ifB9wIPAtsARwH\nHAJ8IyYfERERaS6q40VERPITe8d7O+AKYEdgKfAX4Ch3v6PWgYmIiEhdqY4XERHJSew83p/OKxAR\nERFpHNXxIiIi+alFH++assj07rmEEc9iI89TRCxNcwArOYTZV+hZf1pUzkOGXpA97+7IvIecnz3v\nntOj8o4Re7zPWL8uKv03hw7Lnnd3XN7fGJI97zN74vKOEnkQY86VmHMQKjjHW/U8jDhXYs5BiDvH\nv2ExeTddVVtbHWTexe6Bu4tXH0eMmHMvNu+IIo+thf8QkfaFyMwPWhqROHbMwNiv4ohjuL4jroCG\nDO3JnHYNm0TlHRP32uHl0xQaHpH39rwclfdeL8bF8sKbsqfdOua8Av42OnvacbeXT9NHzHkbWT4x\n32/x34Ur4mIZGpH/8PWRsUSYH5fcH8iedtVecXmPvD4i8Scj0kacU5VMJ7aBmZ1mZj1m9p1q8hER\nEZHmojpeRESkdipueJvZ2wnTi/y5duGIiIhIo6mOFxERqa2KGt5mtjlwJfBpYElNIxIREZGGUR0v\nIiJSe5Xe8f4v4FfuHtuzQkRERJqb6ngREZEaix7xxcymAPsBk2ofjoiIiDSK6ngREZF8RDW8zWxn\n4LvAEe6eeXzKW6bexvAxfYcGHD9lHBO6xsVsXkREpMnNS16FVjcikGiV1vFTr4Axm/ZdNuVA6Dqo\ntvGJiIg00qw3YFbRAPNLu7OvH3vHeyKwLTDXbMMkMB3AIWb2RWC4e//5qY6ccTg7du4QuSkREZFW\nMyF5FXoJ+FEDYolWUR0/4xPQ+dY6RikiItIAXZuHV6G5a2Bixun4YhvetwJ7Fy2bCTwGXJBWIYuI\niEhLUB0vIiKSk6iGt7uvAB4tXGZmK4DX3P2xWgYmIiIi9aM6XkREJD8Vz+NdQFfARURE2pPqeBER\nkRqIHtW8mLu/qxaBiIiISHNRHS8iIlIbFtNly8ymAdOKFs9399Thyc2sE3jgUw+cmH1wtTy7kG0Y\nKyYHkXEP6bggc9qentPzDCVKnoewqcQcxNiDkmMB5XlexWqWQ9iy52zseRK7oz3Z8x8yNPt5Bfmf\nW1nleQhj8n5p7kIum3Q5wER3nxsXVf1UWsffcv9I9unsyLSN523n6oIcwJs84+g2FRhCT1T6X/k/\nZU776aFvjwvmqS9kTjpss7gR9deN2yJ74tf+EJW3PXhgVPoX9tkqc9rY82prX5Q57XKLOCbAGF+S\nOa0T9709xw/InPa4oftH5c0nij/6ZRwekXa3uKw5MfsX7BZ/ejUq68c22ytz2o7Iz/3Ltm3mtEOJ\nGAob2MTXRqVfwablEyUe9n2i8v54x5Tsie29UXnbz0ZmTju+6/6ovB95dt/MaQ95y+2Z0y6f+wRz\nJ50CGer4Su54zyN83Hq/LdZXkIeIiIg0H9XxIiIiOaik4b3e3eMuL4mIiEgrUB0vIiKSg0oGV9vd\nzF4wsyfN7Eoz26XmUYmIiEgjqI4XERHJQWzD+4/ACcBRwMnArsBdZrZZjeMSERGR+lIdLyIikpOo\nhre73+zuv3D3ee5+C3A0sCVwbOyG5816JHaVljRv1qPlE7UBlWf7GDxlqf1sJ4NlP/NUyzr+ulnr\nah5fMxos+8maWY2OoC5mz1rV6BDy5w83OoK6WPfz6xsdQl38etbKRodQFz6nPb6DqppOzN2Xmtnj\nlBmv8JaptzJ89PA+y5a/sJwJXeOr2XxLeOSaR5nQlTogbFt5ZNajKs82MWjKUvvZVpplPx+Z9QiP\nXNP3At3qJWsaFE11stbxZ01dw6gxfUdnfumFHj7YNSzP8JrC9desHxT7yZprYHhXo6PI3a+vWc37\nurKPqtya5gF7NzqI3K37+fUM+5cPNDqM3P36mlX8U1f2Ecxb1j3XwOTGfwe9MutOXrnmd32WrV+y\nIvP6VTW8zWxzQoX804HSHTnjiH7Tif3P+35ezaZFRESazviu8YwvugBQMJ1YS8lax399xvB+04kd\nf8wguHMoIiKDynZdh7Fd12F9lhVMJ1ZW1KPmZnahmR1iZm8xs8nA9cA6oD3u/4uIiAxSquNFRETy\nE3vHe2fgamBr4FXg98Dfu/trtQ5MRERE6kp1vIiISE6iGt7uHvtw/QiARY8t6vfGmqVreGnuwrSN\nRG4igln5NJUqEfeaJen7OSRiN3vSjlN8KDVR6hCWLM9WFVOesedVjgVUi/OqVmUZs5t5HsKWPWdj\nz5MSO1pyP3uy5x9zXkH8d1YtpO1njQ5hqpi8F83fUAeOiImn3iqt4/82v6ffG8uWOH+Z291v+cu2\nuqLYsljk/bdXK0Pov49Qej+f8cXZM/fn44KZ92D2rEdGji+wvsQA9j1LYf3cosz/GpW1PxbXd/ov\n67OX5ysW17VhtKcPirdsiTNvbt/3Vkaes1v4+sxpY38NPO1Lsif2l0q8sTr9vdfm9l82kAURaWM/\nmquzH5nuh15Pf2PZMrof+ku/xQ+PzF4+QyJLaHHEudJR4jullGElztnlS3p4ZO7afstXRTzUvMBL\nHMOSnso1NHI9AAAgAElEQVSe1B+KytmfHp7+xsql+NN9z9FVc+fH5b0w+zFf/toTmdOunP9c73/L\n1vHmOTYCzOyjwFW5bUBERKR1HOfuVzc6iFpRHS8iIrJB2To+74b31oT5QBcA+V3mFhERaV4jgLHA\nze302LbqeBERkex1fK4NbxEREREREZHBLmpUcxERERERERGJo4a3iIiIiIiISI7U8BYRERERERHJ\nkRreIiIiIiIiIjlSw1tEREREREQkRw1peJvZF8zsaTNbZWZ/NLO3NyKOvJjZNDPrKXo92ui4qmFm\nB5vZbDN7Idmf96WkmW5mL5rZSjO7xcx2a0Ss1Si3n2Z2eUrZ/qZR8VbCzE43s/vMbJmZvWxm15vZ\n21LStXR5ZtnPNinPk83sz2a2NHnNMbP3FKVp6bKE8vvZDmXZLlTHtx7V8Rveb/nvEdXxfdK0Q3mq\njqc9yhIa0PA2s48A3wamAfsDfwZuNrNt6h1LzuYB2wM7JK+DGhtO1TYDHgI+D/Sbg87Mvgp8ETgJ\neAewglCum9QzyBoYcD8TN9K3bLvqE1rNHAx8H3gncAQwDPitmY3sTdAm5Vl2PxOtXp7PAV8FOoGJ\nwO3AbDMbB21TllBmPxOtXpYtT3V8y1Idv1Grf4+oju+r1ctTdfxGrV6W4O51fQF/BL5X8LcBzwOn\n1juWHPdxGjC30XHkuH89wPuKlr0ITC34exSwCji20fHWeD8vB65rdGw13s9tkn09qM3LM20/2648\nk/16DTixXcuyxH62ZVm22kt1fOu/VMe31/eI6vj2Ks9kv1THt+irrne8zWwY4SrGbb3LPBzNW4ED\n6hlLHeyePMr0pJldaWa7NDqgvJjZroQrT4Xlugy4l/YrV4DDksea5pvZJWa2VaMDqtIYwpX/xdDW\n5dlnPwu0TXma2RAzmwIMB+5q17Is3s+Ct9qmLFuR6vj21K7fIwNot+8R1fFtUp6q41u/LIfWeXvb\nAB3Ay0XLXwb2qHMsefojcALwV2BH4BzCB2SCu69oYFx52YHwZZdWrjvUP5xc3Qj8Anga+DvgfOA3\nZnZA8gOzpZiZAd8Ffu/uvX0U2648S+wntEl5mtkE4B5gBLCScKX7STM7gDYqy1L7mbzdFmXZ4lTH\nq45vdW31PaI6vj3KU3U80CZlWe+G96Dg7jcX/DnPzO4DngGOJTwqIS3K3a8t+PMRM3sYeBI4DLij\nIUFV5xJgHHBgowPJWep+tlF5zgf2BUYDHwauMbNDGxtSLlL3090fbKOylCanOr59teH3iOr4jVq5\nPFXHt0lZ1ntwtUVAN6FjfKHtgYV1jqVu3H0p8DjQcqMMZrSQ0I9vUJUrgLs/TTivW65szexi4Gjg\nMHd/qeCttirPAfazn1YtT3df7+5PJZXTGYTHzD5Hm5XlAPuZlrYly7LFqY5vT231PRKjlb9HVMf3\n16rlqTo+NW1LlmVdG97uvg54ADi8d1nyeMjhwJx6xlJPZrY54cQY8AuhVSUn/0L6lusowkiTbVuu\nAGa2M7A1LVa2SUV1DPAP7v5s4XvtVJ4D7WeJ9C1ZnimGAB3tVJYlDCE82txPG5Vly1Ad357n2iD4\nHimpVb9HVMeXTN+S5ZlCdXyLlmUjHjX/DjDTzB4A7gOmApsCMxsQSy7M7ELgV4RHz3YCzgXWAbMa\nGVc1zGwzwg8LSxa91cz2BRa7+3OEvjVnmtkTwALg64SRbH/ZgHArNtB+Jq9phD4mC5N0/0G403Fz\n/9yak5ldQpiC4X3ACjPrvVK61N1XJ/9v+fIst59JWbdDeZ5H6Pv0LLAFcBxwCPCNJEnLlyUMvJ/t\nUpZtQnV8C1IdrzqeFitP1fGq42mxsgTqP51Y0v/984STYxWhE/2kRsSR4/7NIpz0qwgn0NXAro2O\nq8p9OpQwTUN30euygjTnEKY1WEn4IOzW6LhruZ+EwR5uInzoVwNPAf8NbNvouCP3MW3/uoGPF6Vr\n6fIst59tVJ4/TmJflezLb4F3tVNZltvPdinLdnmpjm+9l+p41fGtVp6q49unLMvtZ7uUpbtjyQ6J\niIiIiIiISA7qPbiaiIiIiIiIyKCihreIiIiIiIhIjtTwFhEREREREcmRGt4iIiIiIiIiOVLDW0RE\nRERERCRHaniLiIiIiIiI5EgNbxEREREREZEcqeEtIiIiIiIikiM1vEVERERERERypIa3iIiIiIiI\nSI7U8BYRERERERHJkRreIiIiIiIiIjlSw1tEREREREQkR2p4i4iIiIiIiORIDW8RERERERGRHKnh\nLSIiIiIiIpIjNbxFREREREREcqSGt4iIiIiIiEiO1PAWERERERERyZEa3iIiIiIiIiI5UsNbRERE\nREREJEdqeIuIiIiIiIjkSA1vERERERERkRyp4S0iIiIiIiKSIzW8RURERERERHKkhreIiIiIiIhI\njtTwFhEREREREcmRGt4iIiIiIiIiOVLDW0RERERERCRHaniLiIiIiIiI5EgNbxEREREREZEcqeEt\nIiIiIiIikiM1vEVERERERERypIa3iIiIiIiISI7U8BYRERERERHJkRreIiIiIiIiIjlSw1tERERE\nREQkR2p4i4iIiIiIiORIDW8RERERERGRHKnhLSIiIiIiIpIjNbxFREREREREcqSGt4iIiIiIiEiO\n1PAWERERERERyZEa3iIiIiIiIiI5UsNbREREREREJEdqeMugYWYnmFmPmb250bFUo9r9MLNzkvW3\nalQMtWBmp5rZozlv47Nm9oyZDctzOyIiIiLS3tTwlsHEk9cGZnaAmU0zs1ENiqkS/fYjj/XLHJtq\nY6iKmW0BnApckPOmZgKbAJ/NeTsiIiIi0sbU8JbB5KfASHd/tmDZZOBsYExjQqpI2n7kYaBjU68Y\nSvkU0AFck+dG3H0NcAXwb3luR0RERETamxre0vbMbFMAD9YWv92AkKpSYj/yUPLY1DGGUk4AZtcp\nhmuBsWZ2WB22JSIiIiJtSA1vaUpmtnfSh/ifCpZ1JsvuL0p7o5ndk/y/t//yXmZ2tZktBu5O3uvT\nL9nMpgHfSrJZkLzXXfD+m8zsMjNbaGarzWyemZ2YV+xZt1mqf7WZHWZm95vZKjP7m5md1Hs8SoS6\npZnNNLPXzWxJst0RGY9N8bHsPe5/VyrPKmMtXHcssA9wa7m0teDuc4HFwDH12J6IiIiItJ+hjQ5A\npIR5wBLgEODXybKDgR5gXzPb3N3fMDMDDgB+kKTp7Xf8c+Bx4HQ23rkt7pf8C+BtwBTgy8BryfJX\nzWw74F6gG7gIWAS8F/iJmW3h7hfVOvaIbab1Vd8fuBF4ETiL8Nk+K8kjrS+2Ee7kPgWcBnQCnwFe\nTo7ZdaWOTYkYev9fnOenC/KsNNZik5N0czOkrZW5wIF13J6IiIiItBE1vKUpubub2R8IDdZeBwPX\nE+48TgZ+C+wHjCK5q13gQXc/vsw25pnZXELj8peF/ZXN7PuExul+7r4kWfxDM7saOMfMLk36/9Yy\n9vMq3SZwLrAemOzuLyf7cC0wf4BD8IC7n1Swz9sQ+k6f7u4Plzo2ZZTMs8pYC+2Z/Pt04UIz+xiw\nLzDM3f81WXYQMM3djzSzkYSLIl9y999k3Favp4CPRa4jIiIiIgLoUXNpbncDnUmDCeAg4DfAn9nY\nqO29k/yHgvUcuLTKbX8Q+BXQYWZb974IDebRhLu5tYr999Vs08yGAIcDN/Q2ZAHc/SnCneU0acfo\nbmBrM9u8zL6VUjbPCmMttjWw3t1X9i4wsz2AVYQ79YWP5v8jsDD5/xrCnfu9s+5QgdeBkWmPzYuI\niIiIlKM73tLM7gaGAQeY2fPAtsmyCWxsvB4EPFpwh7jX01TIzLYljOR9EunTSDmwXZlsssR+MCH2\npVVucztgJPBEyntpy3oV38V+Pfl3S+CNAdYbSLk8K421nAOBnwFfITyu3+sAkpHP3b3HzC6ksu+9\nwu4KIiIiIiJR1PCWZnY/sJrQV/o54BV3f8LM7gY+Z2abEBqv16Wsu6qK7fY+CXIlYSqpNH8pk0eW\n2A9iY+y12Gas7hLLqxnpPY88i70GDDWzzdx9BYC7XwZgZh8ELkn+3wFMAr5YsO4w+ndLyGJLYOUA\nj/qLiIiIiJSkhrc0LXdfZ2b3ERqvz7KxwXQ3MBw4DtgeuKuazaQsexVYDnS4++0VZRofezXbfIXQ\nyN8t5b3dI/MqlMfd3VrE2tsXfFdCn20Aksfy9yc80g/hkfL17j6vYN1d3P25qIg3buuxCtYTERER\nEVEfb2l6dwPvBA5L/o+7v0ZofH2V0Dis5A5mrxXJv2N6F7h7D2HE8w+Z2fjiFZIBw7LIHHs120zW\nvRV4v5ntULDObsB7Msaapt+xqVaNYr2HcAd9UtHy3YDX3b23T/fuFDy+nvS3X5v8f2cz+xczm5n8\nfaiZnWlmHWb2aNIXvVAnMCdjfCIiIiIifajhLc3ubkKf4F3o28C+izDd1QJ3f7GK/B8gNOLOM7OP\nmdlHkgbaacBLwL1mNsPMPmNmX40cfTs29mq2eQ7hMeo5ZvYVMzsduJOCO8IVKHVsqnUOVcTq7k8n\naY8oeusFwsB0vU/yvB0oHAztJOCq5P+7Aw8RBq0D+CfCiOzdwOzkAgEAZjYR2Aq4IUt8IiIiIiLF\n1PCWZjeH0G94GWFE8F53E+4YV/OYOe5+P3AmsA9wOXA1sK27vwK8A7gM+ADwfeAUwt3fU/OIvZpt\nuvtcwh3jxcB04JOEBu5thEe7o5U6NpXklUOslwH/bGbDC/J9Hjgb+JGZnQPMBm4ys++Z2dnAn5Nj\njLvfAXyCjaOwH8zG8nioaFv/Ajzj7ndm30sRERERkY3MXYP0irQrM7seGOfuezQ6lnJiYjWzUcCT\nwKnufnmF2/s18BHCNGMPufsEMxsNHODuNyVpNgEWAOe5+8WVbEdERERERHe8RdpE8RzTZrY7cDRw\nR2MiKq3aWN19GXAhYfqwSl1NuOv9SeCWZET0D/Q2uhMnEvqFVzsvvIiIiIgMYrrjLdImzOxFYCbw\nFDAWOJnQl7rT3Z9sXGT9tVKsIiIiIiLV0nRiIu3jRmAKsAPh8ek5wNeatCHbSrGKiIiIiFRFd7xF\nREREREREcpTrHW8z2xo4ijA4UUUjK4uIiLS4EYQuFTe7+2sNjkVEREQaIO9HzY9i47y5IiIig9lx\nhEH9REREZJCpqOFtZl8A/p3QP/PPwJfc/U8pSRdAmJB4m6I3bia0yov9iJMqCamJ3USYsrjYJhF5\nrK1RLHkqtZ8x4ro9fIYfZU5bu/OqFvuZp1qcV82+j7FKnVfp30JfjDivLm6J76talGfMeQWN+c5q\n5vN2EXAdJHWiiIiIDD7RDW8z+wjwbeAk4D5gKnCzmb3N3RcVJV8NodG9Y9EbI1KWBelLW1epPR2R\nsqyUVnhKv3SJZhfX8I7bWq3Oq1rsZ55qcV41+z7GKnVepe/nTlF5t8JxqkV5xpxX0JjvrJY4b1vh\ny1xERERyUMk83lOBS939p+4+nzAN0ErCXLgiIiIiIi3BzCaZ2R/M7A0z6zazfRodUysys1PN7NGI\n9J81s2fMbFiecYk0k6iGd/LhmAjc1rvMw7DotwIH1DY0EREREZF8mNlQ4H+BLYF/BY4HnmloUDkx\nswPMbJqZjcoh7y2AU4ELIlabSejH9NlaxyPSrGLveG8DdAAvFy1/mdDfW0RERESkFfwd8GbgQnf/\nsbtf7e5LGx1UTiYDZwNjcsj7U4T2wTUDJTKz08zsIjP7NTAcuAL4txziEWlKeY9qDoQhjIp7CI6u\nx4abwoRGB1An2s/2MRj2EbSf7aZZ9vNhYF7RMnXtFmlS2yf/DtjYNrNN3X1lHeLJk+WY9wnAbHcv\nObKmmR0BfAL4HDAL2By4FjjVzA5z9ztzjE+kKcTe8V4EdLPxi6rX9sDCUisdBUwper03csOta+9G\nB1An2s/2MRj2EZqnoZa3wVKezbKfewNdRa9mHW1dZPAys8uBOwkjcP6vmfWY2e3J49g9ZraXmV1t\nZouBuwvWe5OZXWZmC81stZnNM7MTU/LPlG6A+A4zs/vNbJWZ/c3MTjKzc8yspyDNm83sEjObb2Yr\nzWyRmV1rZm8pymsa8K3kzwXJ/nWb2ZurjdXMxgL7ELqdDuRDwN3ufqe77+juL7r7XGAxcEyWbYm0\nuqg73u6+zsweAA4HZgOYmSV/X1T78EREREREau4HwPPAGcD3gD8Ruk4elLz/c+Bx4HSSu8Vmth1w\nL+Em1EWEG1LvBX5iZlu4+0Ux6Uoxs/2BG4EXgbMIv9fPSvIpnKrj7cDfE+4gPw+MBT4P3GFm49y9\n93GbXwBvI9z7+jLwWrL81WpjJTzC7sDcDOm+m7J8LnBgmXVF2kIlj5p/B5iZNMB7pxPblDBIgoiI\niIhIU3P3e81sBKHhfbe7XwdgZr0N7wfd/fii1c4jNML3c/clybIfmtnVwDlmdqm7r4lIV8q5wHpg\nsru/nMR1LTC/KN2v3f0XhQvM7FfAHwl3mK9K9nWemc0lNLx/6e7PFqT/fpWx7pn8+3Tam2Z2HPDP\nhMeB/tnMDgNOKehL/xTwsQHyF2kb0dOJufu1wL8D04EHCY+XHOXur9Y4NhERERGRenPg0pTlHwR+\nBXSY2da9L+C3hEHLOjOkG12Qrh8zG0J4kvSG3kY3gLs/RbgLTsGyNQXrDTWzrQgN2SUDbSNinwaM\nNbE1sL5UH3h3vwo4H1gHHOvunygawO51YGRyEUSkrVU0uJq7XwJcUuNYRERERESaQZ87uGa2LaFx\nfRLpU2A5sF3WdANsdztgJPBEynt9liWN1a8RBjfbiY0DqDkZxjGuQaxZ7QvMd/f1aWEUbEukrdVl\nVPMf8Rlgx3psqsayfweczfSonKczLTaYQSBuwM3pnJ1THBBT9hdGlv1Xci17jZ7cX9x5dXqu51XM\ngG7FTxQO7JUhZ0Wl364n5jzUeSUig86qor97nxK9kjANVpq/EKbVypKuFi4mjBQ+g/B4+VLCD5j/\nIdtTrVn3aSCvAUPNbDN3X1Eizb7AQyXe2xJYWeZxdpG2EN3wNrODga8AEwmt6fe7++xaByYiIiIi\n0iReBZYDHe5+e6lEyaPiZdMN4BXC1c7dUt7bvejvDwEz3f3Ugu0PJ32u7rQ7Cpn2qYzeq8S70n8u\nxV77UPSYfIFdgccq3LZIS4nu4w1sRrhq9Xn0WIiIiIiItDl37yGMDv4hMxtf/L6ZbROTrsx2bgXe\nb2Y7FKy3G/3nJuym/2/5U9h4171Q793oDY3yamNN3EN4tGzSAGn2o/Qd705gTobtiLS86Dve7n4T\ncBNsmEpMRERERKTdnQYcBtxrZj8CHgW2IjwF+i5gm8h0pZwDvBuYY2b/Tfi9/gXCHeV9C9L9Gjje\nzJYl2ziAMDDbopQ8HyA0kM8zs2sIg53NrjZWd3/azOYBR5Ayw5GZ7ZLk9+eU9yYm790w0DZE2kVd\n+niLiIiIiDShzE9vuvsrZvYO4GzgA8DnCH2cHwFOjU03wHbmmtl7gP8kzCL0PKExvkfy6nUKYdqx\njwIjgN8TGsA3F++Xu99vZmcCJwNHEe6U7+ruz1YTa+Iy4FwzG57SV7sTmOfur6Ws9y/AM+5+Z8bt\niLQ0NbxFREREZNBx999R9Fi2u59LmEe71DqLCA3eU8rknSndAOvfSdHj22Z2PaER3ptmGfDplNXf\nWiLP8whzjNc0VkLD+wzCBYDLk1jPJfTrngT8X/EKZrYJ8PG0eETaVZ0a3jcTLsQVmkDcCL8iIiLN\n7mH6jy+kUeFFJI6ZjXD31QV/7w4cTdKwbSbuvszMLiQMvnx50jf8dOBe4EDgUymrnQisJX2+dJG2\nVKeG91G05nRiIiIiMfZOXoVeAn7YgFhEpIU9ZWYzgaeAsYRHxFcDFzYwppLc/VvAt5L/LzKznxAa\n3Ve5+9Mp6S9FjW4ZZPSouYiIiIhIc7kRmALsAKwhjPz9NXd/sqFRZeTun2t0DCLNppJ5vDcjzC3Y\nO6L5W81sX2Cxuz9Xy+BERERERAYbd097PFtEWlgl83hPAh4kTEvgwLeBuQwwEIWIiIjUj5ltaWZX\nmdlSM3vdzH6cXDjPuv4PzKzHzCodbElEREQKVDKP9++orMEuIiIi9XE1sD1hTt9NCPPrXgp8rNyK\nZvYB4J3AC2XSbU0YxGUBGkFOREQGpxGEcRhuLjFt3gZRDW8zO50wx9+ewCpCf5OvuvvjZdZk45Pp\n5WSeThGAs5meOe10pkXlnT1mmM7ZkXlL9bKXT+x5dWHEefUVlX2biTmvtojK+VqOzZz22Mjzarse\nnYcSmNmehAbxRHd/MFn2JeD/zOzf3X3hAOvuBHwvWf83ZTZ1FHBVbaIWERFpaccRLnqXFHvH+2Dg\n+8D9ybrnA781s73cfVVFIYqIiEgtHQC83tvoTtxKuAL5TuCXaSuZmQE/Bb7l7o+FPwe0AODKybDn\n6L5vTH0AZkzsv0L33VnC38gjrpn+/YqT4jKPuCB7/5d/lLp86myY8b7+yyd97/eZ8z6JgzKnbZSb\ngPfUeZuTI9Ke8LP7o/L++vGTUpdfSf9HQo4+NSpr/A/Z0046Y05c5l/cJHPSk55K38dGlGUjlNrP\nXSLyOPp/47bpP8ue9rnUb+HSSvUTOhtSbxVtNTt73h9+05VRsbx7UtkHp2qumc/bRcB14b8LyqWN\nani7+9GFf5vZCcArwEQgey0jIiIiedmBUDdv4O7dZrY4ea+U04C17n5xxu2shtDo7tyq7xtjhvVf\nBrC+I2POiZiGd/y0pdkz79w5ffmYkaXe2y9z3q0w2eoI6h/nHjGJ9+yMyntsieWbprzXGdNSA3x0\n+TQb7L1/XOYjh2dOWqq8GlGWjVBqP8dG5NE5Lm6bvk32tKPisi6ZfhSwT8ry7SZkz3vk2D2jYmnE\n+dMi523ZLlfVTic2hlBzLa4yHxERERmAmZ0PfHWAJA7sVWHeE4FTgMiWQLi7PWZY32UvrKwkChER\nkeb1MDCvaFnMACcVN7yTR9K+C/ze3R+tNB8RERHJ5D+By8ukeQpYCGxXuNDMOoCtkvfSHARsCzxX\n8Ih5B/AdM/tXd39rqQ3OmNj/7vYxd5aJUkREpMXsnbwKvQT8MOP61dzxvgQYBxxYPulNhIcECk2g\nf+giIiKtrNrr4aUlo6UOOGIqgJndA4wxs98Q6uge4I+EkQPvLbHaVYRK+WBgZ+ANYHPgx0DWR89F\nRESkhIoa3mZ2MXA0cLC7v1R+jffQEk/mi4iIVKXa6+HVc/f5ZvYacCjwOWAkYWDUZwtHNDez+YSZ\nSX4JrCWMO/Q14C/AlsBdwPvd/cuxMUwZW+1etIYp2btyt7SI7qIt7YBGB1AHg6UsB8t+fqDRAdRJ\nu5RndMM7aXQfAxzq7s/WPiQRERGpVDKd2DbAjYQGdw9wG3Ckme1Q0PjeHRgN4O7LCNODFeazCNjF\nzHZ29+djYugaW9UutIyu6B7xrWmwPJ84GBreg6UsB8t+DpaGd7uUZ+w83pcAXcD7gBVmtn3y1lJ3\nr82zdCIiIlKN3unE/rF3QdLHezUF04m5e7kxxk8g9BVbkk+YIiIig8eQyPQnE0auvxN4seB1bG3D\nEhERkQqlTidGmIFkoOnENjCz4cAFwNXu/kbNIxQRERlkYufxjm2oi4iISA3kOZ1Y0XaGAj9P8vt8\nufRp04lNGTt4HjcXEZHBoWHTicXx5FXe2UyPynk6Z1cQTx6sfBJpoLjy+UqO59WFnBsRx7Tc4pBa\nWB6V+ljOikh9aFwoJQerTtc9/vTMaTse0XnYJPKcTqw3XW+jexfgXVnudqdNJyYiItJu6jqdmJmd\nTBghdWyy6BFgurvfFJOPiIiIxKlgOrH9gcnAvwNvItT5awZYbyhwBzCR0BXtXjP7prtfUYPwRURE\nBrXYR8efIzzm1kmomG8HZpvZuFoHJiIiIvHcfT5wM/C/wHeAKwnjsfwNuNrMtoEwnZiZHZP8fyjQ\nO+/3NcC7gMuAH5vZe+q+EyIiIm0mquHt7v/n7je5+5Pu/oS7n0l41vKd+YQnIiIiFfgoYTBUgC8C\ntwL7ASuBTybLN0wnBuwEHEHol/MJ4G7gPKAD1OdFRESkWhX38TazIYTRzIcTKmgRERFpDiuAMcCH\n3H1270Izu5VkuuLC6cTc/Rkzuxt4wN3/rSD9CcCMegUtIiLSrqIb3mY2AbgHGEG4cn6suz9R68BE\nRESkYtsQ7la/XLT8ZWCPEuvsUCL9KDMb7u4l+4eLiIjIwCq54z0f2JfweNqHgWvM7FB3f7D0KjcT\n2umFJiQvERGRdlHtZCOtR9OJiYjIYFD36cTcfT1huhKAB83sHYSRzk8qvdZRwI6xmxIREWkx1U42\nUjOLgG5g+6Ll21N6SrGFJdIvG+hut6YTExGRwaDaGj52VPNSeXSUTSUiIiJ14e7rgAeAqWb2tJmt\nMrM/Au8F5pRYbRHwGTN7xcyWmtkc4OOE7mUiIiJShaiGt5mdZ2YHm9lbzGyCmZ0PHEKYqkRERESa\nxz3AYcBvgQ8SRjnfFpgNYGbnm1nhHN1LgGGEacU+BCwF/gG4oX4hi4iItKfYO97bAVcQ+nnfSpjL\n+yh3v6PWgYmIiEhVDgDuIPT3uo7QkH4VOCZ5fwdgl97E7v4p4N2EJ+l+BewGPE+o+0VERKQKUX28\n3f3TeQUiIiIitWFmwwgXx4unE5vJxunETixez93vStbDzAxYACzOP2IREZH2VvE83gBmdhpwHvDd\nwnk/U1Imr/Kmc3ZsFBFpPSrns5meOe10pkXlLc0u5rzaIirnUyPO8R9wblTeJ+s8bHIx59XvIvP+\neFTqjke+EpH663GhSDOoZDqxYl8BNgOurWFcIiIig1LFDW8zezthJPM/1y4cERERaTQz+yhwFvA+\nd180UFpNJyYiIoNB3acTAzCzzQkDqn2aUDGLiIhI86hkOjEAzGwKYXaUD2cZw0XTiYmIyGDQqOnE\n/j5mck0AACAASURBVAv4lbvfXuH6IiIikpMKpxPDzLqAnwDnAr8ys7l1CVhERKTNRTe8kyvh+wGn\n1z4cERERqZGo6cSSx8uvAM4APg/cBQw1s1H1DVtERKT9xM7jvTPwXeC45Gp6RjcBs4peD8dsWkRE\npAU8TP/67qZGBRM1nRjwGcKAbN8B3kKYw3tvQr0fZdaCCiNuMbMebHQE9TFYfrHd0+gA6mCwlOVg\n2c/rGx1AnbRLecb28Z5IuFo+N5lmBEIlfYiZfREY7u4pQ4e/B9ixijBFRERaQbU9wGqjkunE3P0f\nzOxE4LPAZMIYLse4+ydjt3/NgsExuNo1D0HX/o2OIn/z6H9Wt6N7SD4cbWywlOVg2c/rgQ80Oog6\naJfyjG1430r//Z4JPAZckN7oFhERkTqLnk7MzHYnTBF6kLv3bLy+LiIiItWKani7+wrg0cJlZrYC\neM3dH6tlYCIiIlIfZjYEuAqY5u5P9i7Osm7adGIvrKxpeCIiIg3XkOnEiugut4iISHOJnU5sC2AS\nsJ+Z/VeybAhgZrYWeLe735m2obTpxI5JTSkiItK6qu1MVnXD293fVW0eIiIiUjvuvs7MHgAOZ+Mo\n5pb8fVHKKsuACUXLvkAYYO1DwIKUdUYAzF/a/40l62Du4v7Lu7uzxd8rrgPbS3GZR9w3mPt8+vIl\nq0q991DmvGOjboTV1D/Ov8Yknh83692CEstXprw397morPGUz0NJD0eOzrdqk8xJS5VXI8qyEUrt\nZ0zDZ+6j5dMU8kXZ00aeVmxWYvky4C8py7cqvi07gFWL50fF0ojzp5nP24JiH1EureXZLdvMOoEH\n4CTyG1wte/xnMz0q5+mcHZFafeHayxYRaZfnmHdMWoAXI9NLfWX/vvpd5PfVoUyLDUbqZsP18Inu\nXrd5sc3sWMI4LCcD9wFTgQ8De7r7q2Z2PvAmd/9EifWnEQZX6yzx/kcJj6eLiIgMdse5+9UDJYi6\n451UwsW/7ua7+7jYyERERCQ/7n6tmW0DTCc8Yv4QcJS7v5okKZ5OLNbNwHGEm4Qx3dxERETaxQhg\nLKFOHFAlj5rPIzyq1nuLd30FeYiIiEjO3P0S4JIS752Ytrzg/XOBcwd4/zVgwKv7IiIig8CcLIkq\naXivL7haLiIiIiIiIiIDGFLBOrub2Qtm9qSZXWlm1TymJiIiIiIiItLWYhvefwROAI4iDNayK3CX\nmZUabE9ERERERERkUIt61NzdCzuNzzOz+4BngGOBy0uveRP9R1ifQP+Z0ERERFrZw4ShUApp3DER\nEZHBrqp5vN19qZk9Duw2cMr3kN90Yv+/vTsPl6ss0/3/vROGMCMgREUFAUEOiAQVw2QUWwKeBgUP\nMtgKSiODLYdumRQJRAWFBm2k0wdRCbQCJ+eHCCgQRaFJZGoSQCLEgRkhgTAkGubk+f3xrg2V2lW1\na1XVqtq16v5cV13JXvXWqufNU6ldT613MDMzGy22ZfiXyq9tJ2ZmZmYDqpU53q+RtCap6B6te5qb\nmZlZF0k6WtKDkl6QdKuk9/U6pk6SNEXS8qrbvb2Oqx2SdpV0VbaGz3JJe9doM1XS45Kel/QrSSNc\ndBl9RuqnpAtr5PaaXsXbCkknSbpd0hJJCyVdIemdNdr1dT6b6WdJ8nmEpLslLc5uN0uaXNWmr3MJ\nI/ezDLmEnIW3pLMk7Sbp7ZJ2Aq4AXgEuLSQ6MzMz6xuSPgWcDUwBtgfuBmZm+4mXyTzS3ujjs9su\nvQ2nbWuQ9nk/CojqOyWdAHwROBx4P7CUlNdVuhlkBzTsZ+ZaVsztgd0JrWN2Bb4H7Ah8BFgZ+KWk\n1YYalCSfI/Yz0+/5fBQ4AZgA7AD8BrhK0tZQmlzCCP3M9Hsucw8135i0Z+f6wFPAbOAD2V6eZmZm\nNtiOBc6PiIshXcUAPgZ8Djizl4F1WKm2Vo2I60gL8iBJNZocA3w9In6etfkMsBD4ODCjW3G2q4l+\nArzUz7mNiL0qf5Z0CPAkqZiZnR3u+3w22U/o/3z+ourQyZKOJH3hcC8lyCU01U/o81xCziveEXFg\nRGwcEatFxNsi4qCIeLCo4MzMzKw/SFqZ9KH310PHIiKA64GJvYqrIAOztaqkTUlXlyrzugS4jfLl\nFWBSNnR5vqRpktbrdUBtWpd0df8ZKHU+V+hnhdLkU9IYSQcAq5J2lSplLqv7WXFX3+cy9+Jqkt4M\nfBvYE1gd+BNwaETM7XBszUbUdMupnFLYua0X6o0SG+4UpuY681Sm5A0mh78W1NY6o/nX1Xdyvq6O\nzfG6+mDu9yuzntsAGEu62lJpIbBl98MpzNDWqn8grRx7KulD8DYRsbSHcRVlPOmNsVZex3c/nEJd\nC1wOPAhsBpwBXCNpYvYlUl/Jrup/F5gdEUNXDUuXzzr9hJLkU9I2wC2kLaKeB/aPiPslTaREuazX\nz+zuUuQyV+EtaV3gt6RvVvYAFgFbAM92PjQzMzOz0aX1rVVttIuIyqG5v5d0D3A/MAm4oSdBtWca\nsDWwc68DKVjNfpYon/OB7YB1gE8Cl0n6YG9DKkTNfkbEnWXJZd5VzU8EHomIwyJiTkQ8HBHXe7i5\nmZnZwFsELCMtflNpI2BB98PpjohYDDSxtWrfWkAaAjhQeQXIPt8uog9zK+k8YC9gUkRU7j5Uqnw2\n6Ocw/ZrPiHg1Ih7ICtCvkoaSH0nJctmgn7Xa9mUu8xbefw/cIWlGNsZ+rqTDigjMzMzM+kdEvALM\nAXYfOpYNAd0duLlXcRVNJd9aNfuAu4AV87o2adGj0uYVQNLGpAWF+yq3WTG6D/ChiHik8r4y5bNR\nP+u078t81jAGGFumXNYxhjR9aZh+zWXeOd7vIH3zcDbwTdKy9edKeiki/rPTwZmZmVlfOQeYLmkO\ncDtplfPVgem9DKqTJJ0FXE0aXv4W4DT6fGtVSWuQvjwYWtzmHZK2A56JiEdJ82dPlvRn4CHg68Bj\nwJU9CLdljfqZ3aaQ5pEuyNp9mzSaYebws41OkqaRtlnaG1gqaehq6OKIeDH7e9/nc6R+ZrkuQz5P\nJ81vfgRYCzgY2A34Rtak73MJjftZllxC/sJ7DHB7RHwt+/nubCL8EUDOwvseYNucT9+P3M9yGYR+\nDkIfwf0sm0Hp5+gWETOyPbunkoY73gXs0e9bwFQp49aq7yXNk4zsdnZ2/CLgcxFxpqTVgfNJq0fP\nAvaMiJd7EWwbGvXzKODdwGdIfXyc9KH+lGw0R784gtS3G6uOHwpcDFCSfI7Uz2WUI58bkl6fbwIW\nA78jvafeAKXJJTTop6RxlCOXuQvvJ4D7qo7dB+zb+GHXkRaoq7SEwfiQNA/3s0wGoZ+D0EdwP8tm\ntPTzHlIslV6s1bC0ImIaabGjUoqIA3sdQ6dFxH8xwvTDiDiVtIJ732qin5O7FUtRIqKpaaT9ns+R\n+pld3S9DPkec0tvvuYTG/SxLLiF/4f1bhm8JsiVpuFUDk0lfYFTq2xFZZmZmdWzL8C8AngC+34NY\nzMzMbLTIu7jad4APSDpJ0maSDgIOA87rfGhmZmZmZmZm/S9X4R0RdwCfIC1mcA/wVeCYiLisgNjM\nzMzMzMzM+l7eoeZExDXANU02zyZ2L6px14t0fwX4yNleIzcZUS/62QujO5/5I6v3iEHI5yD0Eer3\ns/nX1aO5nzPPv2un3q8GPZ+jwWu/A6sXOzEzM7MBoYi8H+5ynDwNRf9JYU9gZmbWPw6OiEt6HYSZ\nmZl1X9GF9/rAHqR95QZrWVczM7NkHLAJMLPPt5wyMzOzFhVaeJuZmZmZmZkNuryrmpuZmZmZmZlZ\nDi68zczMzMzMzArkwtvMzMzMzMysQC68zczMzMzMzArkwtvMzMzMzMysQD0pvCUdLelBSS9IulXS\n+3oRR1EkTZG0vOp2b6/jaoekXSVdJekvWX/2rtFmqqTHJT0v6VeSNu9FrO0YqZ+SLqyR22t6FW8r\nJJ0k6XZJSyQtlHSFpHfWaNfX+WymnyXJ5xGS7pa0OLvdLGlyVZu+ziWM3M8y5NLMzMzKq+uFt6RP\nAWcDU4DtgbuBmZI26HYsBZsHbASMz2679Dactq0B3AUcBQzbg07SCcAXgcOB9wNLSXldpZtBdkDD\nfmauZcXcHtid0DpmV+B7wI7AR4CVgV9KWm2oQUnyOWI/M/2ez0eBE4AJwA7Ab4CrJG0NpckljNDP\nTL/n0szMzEqq6/t4S7oVuC0ijsl+FukD1bkRcWZXgymIpCnAPhExodexFEHScuDjEXFVxbHHgbMi\n4jvZz2sDC4HPRsSM3kTanjr9vBBYJyL27V1knZV96fUksFtEzM6OlTGftfpZunwCSHoa+HJEXFjG\nXA6p6mcpc2lmZmbl0NUr3pJWJl2p+PXQsUiV//XAxG7G0gVbZMOV75f0Y0lv7XVARZG0KenqUmVe\nlwC3Ub68AkzKhi7PlzRN0nq9DqhN65Ku7j8Dpc7nCv2sUJp8Shoj6QBgVeCmsuayup8Vd5Uml2Zm\nZlYuK3X5+TYAxpKutlRaCGzZ5ViKdCtwCPAH4E3AqaQPwdtExNIexlWU8aSCplZex3c/nEJdC1wO\nPAhsBpwBXCNpYnR7+EgHZCNOvgvMjoihdQhKl886/YSS5FPSNsAtwDjgeWD/iLhf0kRKlMt6/czu\nLkUuzczMrJy6XXgPhIiYWfHjPEm3Aw8D+wMX9iYq64Sqobm/l3QPcD8wCbihJ0G1ZxqwNbBzrwMp\nWM1+liif84HtgHWATwKXSfpgb0MqRM1+RsSdJcqlmZmZlVC3F1dbBCwjLX5TaSNgQZdj6ZqIWAz8\nEei7lYSbtAAQA5ZXgIh4kPS67rvcSjoP2AuYFBFPVNxVqnw26Ocw/ZrPiHg1Ih7ICtCvkoaSH0nJ\nctmgn7Xa9mUuzczMrJy6WnhHxCvAHGD3oWPZENDdgZu7GUs3SVqT9OGv4Yf+fpV9wF3Ainldm7Sa\ndGnzCiBpY2B9+iy3WTG6D/ChiHik8r4y5bNRP+u078t81jAGGFumXNYxhjR9aZgS5dLMzMxKoBdD\nzc8BpkuaA9wOHAusDkzvQSyFkHQWcDVpePlbgNOAV4BLexlXOyStQfryQNmhd0jaDngmIh4lzZ89\nWdKfgYeArwOPAVf2INyWNepndptCmke6IGv3bdJohpnDzzY6SZpG2mZpb2CppKGroYsj4sXs732f\nz5H6meW6DPk8nTS/+RFgLeBgYDfgG1mTvs8lNO5nWXJpZmZm5dX1wjsiZmTb+kwlDXe8C9gjIp7q\ndiwF2hi4hHS15SlgNvCBiHi6p1G1572keZKR3c7Ojl8EfC4izpS0OnA+afXoWcCeEfFyL4JtQ6N+\nHgW8G/gMqY+Pkz7Un5KN5ugXR5D6dmPV8UOBiwFKks+R+rmMcuRzQ9Lr803AYuB3pPfUG6A0uYQG\n/ZQ0jnLk0szMzEqq6/t4m5mZmZmZmQ2Sbi+uZmZmZmZmZjZQXHibmZmZmZmZFciFt5mZmZmZmVmB\nXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZ\nmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZ\nFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHib\nmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZ\nmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciF\nt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHibmZmZ\nmZmZFciFt5mZmZmZmVmBXHibmZmZmZmZFciFt5mZmZmZmVmBXHib5SDpVEnLexzDIZKWS3pbi48/\nNXv8er2KoRMkHS/p3gLO+wVJD0taudPnNjMzM7PB5MLbLJ/Ibv0cQ1OPlzRR0hRJaxcQQ1skrQUc\nD3yrgNNPB1YBvlDAuc3MzMxsALnwNus/FwOrRcQjBT/PTsApwLo9jKGezwNjgcs6feKIeAm4CPjn\nTp/bzMzMzAaTC2/ra5JW73UM3RbJy114Ko2CGOo5BLiqwBhmAJtImlTQ+c3MzMxsgLjwtr5RMTf5\nXZIukfQMMCu7782SfiRpgaQXJc2TdGiNc7xN0jRJ8yU9L2mRpBmS3l6j7S6S/lvSC5L+JOnwJuPc\nNovzf1Ycm5Adu6Oq7bWSbqn4ecR+1JtfLWmSpDsq4x1hTvobJE2X9Kyk57LnHZedawpwZtbuoez5\nlg09Z3UMFbnZrN4524y18rGbAO8Grh+pbasiYi7wDLBPUc9hZmZmZoNjpV4HYJbD0Jzi/wf8ETgJ\nkKQNgduAZcC5wCJgT+CHktaKiHMrzvE+4APApcBjwCbAUcANkraOiBdJJ90GmAk8SRpuvTJwavbz\nSOYBzwG7AT/Pju0KLAe2k7RmRPxNkoCJwP/JnrPZfgybXy1pe+Ba4HHga6T/21/LzlFrLrZIV3Uf\nAE4EJgD/CCwk/bv+FHgncABwDPB09rin6sQw9Pfqcx5Wcc5WY622U9ZubhNt2zEX2Lng5zAzMzOz\nAeDC2/rRnRHxD0M/SPoBqZB8T0Q8lx3+vqRLgFMlnZ/N2wX4eURcXnkySVcDtwL7AT/JDn89+3OX\niPhL1u5yUlHdUESEpN+Siu0huwJXkK6g7gT8EngPsDbZVXvg9Bz9qHYa8CqwU0QszOKdAcxvEOqc\niHjtKr6kDUhzp0+KiHskzSUV3lfmmMtd95xtxlppq+zPBysPSvo0sB2wckT87+zYLsCUiPg7SauR\n8vdPEXFNE8/zAPDpJmMyMzMzM6vLQ82t3wRwftWxfYGrgbGS1h+6kYrbdUhXXtODKwpXSStlW2o9\nQLpCPSE7Pgb4KHDFUNGdPfYPpKvgzZgFTMiKPYBdgGuAu3m9IB+6Cj47bz8qZfHuDvxsqJDN4n2A\ndGW5llr/jrOA9SWt2WQfc5+zxVirrQ+8GhHPDx2QtCXwAulKfeXQ/I8BC7K/v0S6cr9tk8/zLLBa\nraHyZmZmZmZ5uPC2fvTalU5JbyStun04aRh05e1HWbMNK9qPkzRV0iOkQmwRafj4OtkN4I3AasCf\nazz3H5qMcRZpePpESe/MzjkLuIkVC+97I2Jx3n5U2bBBvLWODam+iv1s9ucbGjxmJCOds9VYR7Iz\ncBXwIdJw/SETgd8CRMRy4KwaMdYztLhcr7ePMzMzM7M+56Hm1o9eqPj70JdHPyZtAVXL7yr+fh7w\nWeA7pOHli0mF1f+ls19E3QG8SJrn/SjwZET8WdIs4EhJq5Cugv+0xX50wrI6x+uuZt6jc1Z7GlhJ\n0hoRsRQgIn4EIGlfYFr297HAe4EvVjx2ZV4f2j+SNwDPNxjeb2ZmZmbWFBfe1u+eAv4KjI2I3zTR\nfj9gekQcP3RA0qqsuFf1U6Tifosaj9+qxrFhIuIVSbeTCu9HeL3YmwWsChwMbES6At5KPyo9SSry\nN69xX60+NKuIK72diHVoLvimVMy5z4blb08a0g9pSPmrEVE5L/+tEfFok8+zKXBfk23NzMzMzOry\nUHPra9nw4cuB/ST9j+r7s8W9Ki1j+Ov+S8DYqnPOBD4uaeOKc72LNPe7WbOAHYFJ2d+JiKdJheMJ\npMJ26Hjefrwme+z1WbzjKx6zOTA5R7zVlmZ/rtuwVQ4divUW0hX091Yd3xx4NiKG5nRvQcXw9Wy+\n/cvZ3z8s6VhJv5B0dLaQXLUJwM1NxmRmZmZmVpeveFsZnEgqbm+TdAFwL7AesAPwYaCyaP058A+S\nlmTtJpIW+1pUdc4ppEJwtqRppCHKXyRdYX13k3HNAr4KvJUVhzffBHwBeDAiHm+xH9VOJX0pcLOk\n/yD93z46i3e7JuOtNodU4J4u6TLgFeCqiHih8cNG1FasEfGgpHnAR4DpFXf9hbQw3UoR8Spp67jK\nhdEO5/VV658gLdL2x4j492xl/NdI2oH0b/+z3L0zMzMzM6viwtv6XkQ8Ken9pP22PwEcSZoH/Hvg\n+KrmXyJtZXUQqSibTSrgZlIxtDrbTuujwDmk7a8ey87/ZpovvG8mXWH/G2k18yGzSEXgTZWNc/aD\nqsfOlTQZ+FdgahbvqcCW2S23iLhD0snAEcAepJECm9L84mRFxvoj4DRJqw7NwY6IxySdAlwg6WHS\nYmtjJP0b6d/xpoh4Mmt7n6QvkRXiNeZx/y/g4Yi4sfWempmZmZklivCCvWZlJekKYOuIaKn47qY8\nsUpaG7gfOD4iLmzx+eYC74uIZVXHVwEeAk6PiPNaObeZmZmZWSXP8TYrier9piVtAewF3NCbiOpr\nN9aIWELaGuy4Fp9/feAP1UV35lDSXPDqPcnNzMzMzFriK95mJSHpcdKc5weATUhDxFcGJkTE/b2L\nbLh+itXMzMzMrF2e421WHtcCBwDjgZdIc8y/MkoL2X6K1czMzMysLb7ibWZmZmZmZlYgz/E2MzMz\nMzMzK1ChQ82zBYz2IK0Q/GKRz2VmZjZKjSOtZTAzIp7ucSxmZmbWAy0V3pKOBr5Mmp95N/BPEfHf\nNZruQbZPrpmZ2YA7GLik10GYmZlZ9+UuvCV9CjgbOBy4HTgWmCnpnRGxqKr5Q+mPfYENqu66Dpic\n9+lHsXpz5WeSvn9Y0T9yQdNnvoDDWwupqzqRz1Vytn85R9uxOc9da5cp6P7rNt8aDJ15XfXi/2aR\nua937p8D/7PNc3fqdVXkuevls/nXVp7XFfTqPasTr9utcrRdI0fbR4Bvwmu/E83MzGzQtHLF+1jg\n/Ii4GEDSEcDHgM8BZ1a1zYaXbwC8qequcTWO9bN6H2Jr9zNfz/vh36kT+Rw3cpMV5Jm9kPel/mqd\n491+3eYrvDvzuurF/80ic1/v3KsBb2nz3J16XRV57nr5bP61lf/V0Iv3rE68bjfL0XbtVp7AU67M\nzMwGVK7F1SStDOwA/HroWKRl0a8HJnY2NDMzMzOz4SQdImm5pLfV+nk06ocYRwtJx0u6t4Pn+4Kk\nh7Naxqwn8q5qvgFp/OPCquMLSfO9zczMzMyKFqw4dKf6556QNFHSFEm1hsWMihg7YYR+tnvutYDj\ngW918LTTSXPPvtDBc5rlUuiq5q+7juHDPZd056nNzMy65tfAb6qO/a0XgZhZb+wEnAJcyPAPuxcD\nl0ZEnsVERqtG/WzX50kX+i7r1Akj4iVJFwH/DJzXqfOa5ZG38F5EWsFno6rjGwEL6j9sMsPn3t2T\n86n71Ta9DqBL3M/yGIQ+Ary71wF0yaDkc7T0c/fsVumP+CKL2cBQvTuy6ZllKLqhQT874BDgqgK+\noJgBHC9pUkTc2OFzm40o11DziHgFmEPFpwpJyn6+Od9Tb5uved8aLR8GizYo+RyEfg5CHwHe0+sA\numRQ8jko/TQrN0lvkzRN0nxJz0taJGmGpLe3eL7tJV0rabGkv0q6XtKONdq9WdIPJf1F0ouSHsji\nWKnZuCRN4fWFhh/K5nMvG2keejMxSjo1e+xmkqZLelbSc5J+JKmpFUolTZJ0h6QXJP1J0uFD561o\n03Y/K/49fyRpQfbvOU/SoU3EuAnpm/Hrm+lTHhExF3gG2KfT5zZrRitDzc8Bpkuaw+vbia1Omjth\nZmZmZtaq9wEfAC4FHgM2AY4CbpC0dUQ0vTuApK2Bm4DFpPnCr5KGn9woabeI+O+s3ZuA/yZtV3A+\n8AfSthefJH3GXdJkXJcD7wQOAI4Bns5CeSr7c9gc72ZjrHjcDOAB4ERgAnAYaa2lk0b4t9geuBZ4\nHPgaqQb4Gmk0a2VMbfdT0obAbaRRsudmz7En8ENJa0XEuQ1C3SmLZ26j/rRhLrBzQec2ayh34R0R\nMyRtAEwlDTG/C9gjIp5q/EgzMzMzs4Z+HhGXVx6QdDVwK7Af8JMc5/om6bPuzhHxcHau/yQV1mcC\nH8rafQvYEHh/RNxZ8fhT88QVEfMkzSUVpFdGxCMdjHHInIg4vCKGDUhzohsW3sBppKJ+p4hYmD12\nBjC/ql0n+nk6aSj6eyLiuezY9yVdApwq6fyIeKlOnFtlfz5YryOSDgAmkb4YeAw4MSK2qte+ygPA\np5tsa9ZReVc1ByAipkXEJhGxWkRMjIg7Oh2YmZmZmQ2WyoJM0kqS1iMVS8+RrvA2RdIY4O+AK4YK\n2uz8C4BLgF0krZlNmdyHNKf4ztpn61xcrcRYGQbpinylWcD6Ve1qPc/uwM+Giu7seR4gXQWn4lgn\n+rkvcDUwVtL6Qzfgl8A6I5xnfeDViHi+Tl++APwLcFREfAP4MOmKerOeBVZrdni+WSd1aVXzQZBv\njYmpnFJQHEVrfheMs5ia68zHMSVnLHneM5semZbJ81/j1ZznzmM0va7y7YByXY78T86d+zz6NfdF\nnhvyvLZG1/tV86/DZXPyvQeN3SHP6zDPVrCP54rDbJBlBdFXSAtsvYXX36yCVLQ1642kYeJ/rHHf\nfaSLT28lDZNeG/h9l+JqJcb7Ko5XX11+NvvzDdTfQmFDYDXgzzXuW+FYu/2U9EZgXeBwaq8qGVk8\nuWVTAs4hjbQdmpf+FOmqd9OnqYjDrKtyF96SdgWOA3YgLVX+8Yi4qtOBmZmZmdnAOQ/4LPAd0vDm\nxaQi6f/S4kjNksW1rM7xTq0y3m4/h9r8GLioTpvfNXj808BKktaIiKVV930e+GtEzK44tjNpGH2z\n3gA832Cou1lhWrnivQZpXvcPgZ92NhwzMzMzG2D7AdMj4vihA5JWJV1FzeMp4Hlgyxr3vQtYDjwK\nLCUtnjbSNjTNxpXnSmqzMbbrSdLwr81r3LdF1c/t9vMp4K/A2Ij4TQuxDs053xSYV3XfTsBvaONS\nkAAAG6lJREFUK+IaB2xXeawJm7LiCAKzrsn9DV1EXBcRp0TElRS7h5+ZmZmZDZZlDP98+iVgbJ6T\nZEORfwnsU7XN1UbAgcCsiPhbtrf2z4C/l9Ro7nGzcQ1dpR3xi4JmYxzpPE0+z/XAxyWNr3iezYHJ\nVc3b6mf2XJcD+0n6H9WxZIvBNXILqb54b437VmXFYfnvB+ZHxJLs3FtJmirp7yVdLmm7GueYQO4t\nkM06w3O8zczMzGy0+DnwD5KWAPcCE0kLg+VZQGvIycBHgN9KmkYqKg8HVgGOr2j3FdIiZzdJ+j7p\niuibSduJ7ZwVds3GNYdUOJ4u6TLgFdLCbS+0GWO7TgU+Ctws6T9INcDRpKvKlQVqJ/p5ImnV8dsk\nXZCdZz3SNNUPA3WL74h4UNI80r/J9Kq7byLNi0fSKsDXSYvLIWl10oJ0u0bEUkknZ8/7Gkk7ZHH8\nrN7zmxXJhbeZmZmZjRZfIq0ueRBpFdXZpCJsJjkXxIqIe7O1ic4gFYNjSPOWD6rckSciHpe0I6mQ\nO4i02NpfgGtIQ8Gbjisi7siKviOAPbLn3JThi6LlirFdETFX0mTgX0lbAj9GKsa3ZMWh7m33MyKe\nlPR+4BTgE8CRpLnbv6e5LxN+BJwmadWqudj/CvxA0olZLGuRFd6kldTvyoru9YHnIuKVqvP+L+Dh\niLixiRjMOk5phE2LD5aW02BxtWzIzhx4G8NXoN4G2Lbl5+5/ef7dR9OIfq9qPlzRq0/nUeTrql9X\nNc+rX3NfpLy/J4p8zxrtq5rfDdxTdexF4CGAHSJibq6gzMwKJOkKYOuIqDXPvCckrQ3cDxwfERfW\nabMmsADYLCIWSjoWWB4R/ybp08AmwG0R8aus/SqkN+LTI+K8LnTDbJguXfGeTFoA3czMrMy2Y8VR\nm5C2E5vWg1jMzF4naVxEvFjx8xbAXkDN4rZXImKJpLNIuyhdCCDpaGBCRHw+a/Zl4MKKfckvA46X\ntCewZnarnJd+KPAyw/dBN+saDzU3MzMzMyu/ByRNBx4gXRE+gjQk56wexlRTRJwJnFlxaA1gLUn/\nQpp//2RE/FNF+yeAYxuc73xcdFuPtbKP9xqk7QiGxhK+I1s18JmI6MSWB2ZmZmZm1lnXAgcA44GX\nSKt7fyUi7u9pVE3ICnGzvpZ7OzHS8v53klYzDOBsYC75Nq83MzOzgkh6g6SfSFos6VlJP8i+OG/2\n8f9H0nJJXyoyTjPrnoj4fES8IyJWj4g3RMTHIuLuXsdlNihyX/GOiP+itYLdzMzMuuMSYCPSNkCr\nkLblOR/49EgPlPQJYEfSqs6N2q1PWs34IfKvYGlmZlYG40hTN2ZGxNONGuYqvCWdRNoWYCvgBdIQ\nlRMi4o8NH2g15Fn1N9+KwnlWE8+/knjzcR/HKTnPnVeRn/P6dbXq4l5XP8y5Sv3kwvPfrDyr34Pr\nh1qKWwH/4Zyvq7fneM8au0ORr8HqXWoa6e77iaStSAXxDhFxZ3bsn4BfSPpyRCxo8Ni3AP+WPf6a\nEZ5qD+AnnYnazMysrx1M+tK7rrxXvHcFvgfckT32DOCXkt4VES+0FKKZmZl10kTg2aGiO3M96RuR\nHYEraz1IkoCLgTMj4r70Y0MPQdo8d4OqO64j7WdSFvW+SppJ+vah2gXjc2y/vCDfek+Hc0HTbfNu\nBHgBh9e5pxMZzRdNnn52Srdft7nzs2aO19Xfvl/njnq9zBfNl3PkZ4dcZ4YD674Oa8n3v7MXr6si\ndSKbF3BjzmdtWFu2EQlcWic/Q0vcVzrw5Jzb3H+j3v+Jdi0CfgrZ78RGchXeEbFX5c+SDgGeJP2f\nmp3nXGZmZlaI8aTfza+JiGWSnsnuq+dE4OUce9y+CKnort4wdFyNY/2s3kfHuv1cdUKOs+f7l8rT\nOm9hV//snchovmh68frp9us2d37GduJ1Va+X+aJ5a46278p1ZujMq7x2P8v0vgSdyuZ7cj7rDTna\n5ouk3mtlrVr3vT3P/wfoQvZHHDLZ7nZi65L+RZ9p8zxmZmbWgKQzgBMaNAla+Yybzr0D8CVg+7yP\nvY7hEzqWtBKEmZnZqHYPMK/qWPNTFFsuvLMhad8FZkfEva2ex8zMzJryr8CFI7R5AFgAbFh5UNJY\nYL3svlp2Ad4IPFoxxHwscI6k/x0R76j3hJMZfh3h0hGCNDMz6z/bZrdKTwDNDWNv54r3NGBrYOeR\nm9b6PnwbhgduZmbWz9r7NryRbLXUhiumAki6BVhX0jWk39HLgVtJq+TdVudhPyH9Ut4V2Bj4G7Am\n8AOg2aHnZmZmVkdLhbek84C9gF0j4omRH1Hr+3AzM7Oyae/b8E6IiPmSngY+CBwJrEZaGPWRyhXN\nJc0n7UxyJfAyadrmV4DfAW8AbgI+HhHH5I1hm7Z70R8GpZ+D0lP3skwGo5+D0cvyLNaZu/DOiu59\ngA9GxCOdD8nMzMxalW0ntgFwLangXg78Gvg7SeMriu8tgHUAImIJVUsAS1oEvFXSxhHxWJ4YBmU8\n26B86B2UjLqXZTIY/zsHJZt79jqADsm7j/c04EBgb2CppI2yuxZHhDe/NTMz672h7cQ+NnQgm+P9\nIhXbiUXE2BHOcwhprthzxYRpZmY2OMbkbH8EsDZwI/B4xW3/zoZlZmZmLaq5nRhpB5JG24m9RtKq\nwLeASyLibx2P0MzMbMDk3cc7b6FuZmZmHVDkdmJVz7MS8P+y8x01Unsvn2pmZoOhR9uJWbua31D+\nLKbmOvNxnJI3mIJo5CbWQ/ny8/kCX1fzOS1X+62YkqN1kbNg8r6FvlpIFKNP86+tt+d+XVWXeI28\nkOvMP152V9NtPz32vTnOPNKI7qYVuZ3YULuhovutwIebudrt5VPNzGwwdHE7MUlHkFZI3SQ79Htg\nakRcl+c8ZmZmlk8L24ltD+wEfBl4M+l3/ksNHrcScAOwA2kq2m2SvhkRF3UgfDMzs4GWd+j4o6Rh\nbhNIv5h/A1wlaetOB2ZmZmb5RcR8YCbw/wHnAD8mrcfyJ+ASSRtA2k5M0j7Z31cChvb9vgz4MPAj\n4AeSyrKTi5mZWc/kKrwj4hcRcV1E3B8Rf46Ik4G/klZJNTMzs9HhINJiqABfBK4H3gM8D3wuO/7a\ndmLAW4CPkOYJfBaYBZxOGiefZ26HmZmZ1dDyHG9JY0irma9K+gVtZmZmo8NSYF1gv4i4auigpOtJ\n242tsJ1YRDwsaRYwJyL+uaL9IcB3uhW0mZlZWeUuvCVtA9xCWuHmeWD/iPhzpwMzMzOzlm1Aulq9\nsOr4QmDLOo8ZX6f92pJWjYi688PNzMyssVaueM8HtiMNT/skcJmkD0bEnfUf4s1GzMxsEPwuu1Uq\ncmX93vNveDMzGwxd3k4sIl4lbVcCcKek95NWOj+8/qO82YiZmQ2Cd2e3So8D/9HtQBYBy4CNqo5v\nRP0txRbUab+k0dVu/4Y3M7PB0N52YnlXNa93jo5tUmpmZmbtiYhXgDnAsZIelPSCpFuBPYGb6zxs\nEfCPkp6UtFjSzcBnSNPLzMzMrA25Cm9Jp0vaVdLbJW0j6QxgN9JWJWZmZjZ63AJMAn4J7Eta5fyN\nwFUAks6QVLlH93PAyqRtxfYDFgMfAn7WvZDNzMzKKe8V7w2Bi0jzvK8n7eW9R0Tc0OnAzMzMrC0T\ngRuAPYCfkgrpp4B9svvHA28dahwRnwc+ShpHdzWwOfAY6Xe/mZmZtSHXHO+IOKyoQMzMzKwzJK1M\n+nK8ejux6by+ndih1Y+LiJuyxyFJwEPAM8VHbGZmVm4t7+MNIOlE4HTgu5X7floz1HTL4zilsHND\n5DrzWUxtuu1xTMl1bhvt8ryu8nlXztf4sjee1nTbsU/lfR3meVt8dRSdu1/lfV3lWSF8Uq4zf3rs\nvjlaX5+j7aq54uiQVrYTq3YcsAYwo4NxmZmZDaSWC29J7yOtZH5358IxMzOzXpN0EPA1YO+IWNSo\nrbcTMzOzwdDl7cQAJK1JWlDtMNIvZjMzMxs9WtlODABJB5D2RvlkM2u4eDsxMzMbDL3ZTuzfgasj\n4jctPt7MzMwK0uJ2Ykg6EPghcBpwtaS5XQnYzMys5HIX3tk34e8BTup8OGZmZtYhubYTy4aXXwR8\nFTgKuAlYSdLa3Q3bzMysfPLu470x8F3g4OzbdDMzMxudcm0nBvwjaUG2c4C3k/bw3pb0ez+Xe1oO\nub9Uz/Qrr8HIqHtZJoPxv3NQsnltrwPokLxzvHcgfVs+N9tmBNIv6d0kfRFYNSJqLJPtpVfMzGwQ\n3JndKuVZib0zWtlOLCI+JOlQ4AvATqQ1XPaJiM/lff55DMZv+HmkTzPlNxgZdS/LZDD+dw5KNq8j\nzZPqd3kL7+sZnt/pwH3At2oX3eClV8zMbDBsn90qPUYLF43blXs7MUlbkLYI3SUilr/+/bqZmZm1\nK1fhHRFLgXsrj0laCjwdEfd1MjAzMzPrDkljgJ8AUyLi/qHDzTy21pi2JR2MzczMbHTowXZiVepc\n5TYzM7Meybud2FrAe4H3SPr37NgYQJJeBj4aETfWeqJaY9oubTFoMzOz0au97cTaLrwj4sPtnsPM\nzMw6JyJekTQH2J3XVzFX9vO5NR6yhOETIo8mLbC2H/BQjceMg1ThV3uR9FGkLOpdYajbz5fy7MKW\n718qT+v8V0bqnb0TGc0XTS9eP91+3ebOz7JOvK7q9TJfNI/maJt/SGwnXuW1+1mm9yXoVDbvyvms\nxb0L1Xut/LXWfQ/n3e2yqOy/9luwevDXMKo7LbsDJE0A5sDheI53NzWf07OYmuvMx3FKjtaeH1gu\nI76fVMmzoNRaOc+9Wo62T+Y8t3VXvt9Byw5o/j1r7GVT8gZTkNe+Dd8hIrq2L7ak/UnrsBwB3A4c\nC3wS2CoinpJ0BvDmiPhsncdPIS2uNqHO/QeRhqebmZkNuoMj4pJGDXJd8c5+CVd/kpkfEVvnjczM\nzMyKExEzJG0ATCUNMb8L2CMinsqaVG8nltdM4GDS1fDuL91uZmbWe+OATUi/ExtqZaj5PNJQtaHL\nma+2cA4zMzMrWERMA6bVue/QWscr7j8NOK3B/U8DDb/dNzMzGwA3N9OolcL71Ypvy83MzMzMzMys\ngTEtPGYLSX+RdL+kH0tqZ5iamZmZmZmZWanlLbxvBQ4B9iAt1rIpcJOkNTocl5mZmZmZmVkp5Bpq\nHhGVk8bnSbodeBjYH7iw/iOvY/iKyNswfB80MzOzfnYPaSmUSl53zMzMbNC1tY93RCyW9Edg88Yt\nJ+PtxMzMrPy2ZfiXyq9tJ2ZmZmYDqpU53q+RtCap6C7bfvRmZmbWAklHS3pQ0guSbpX0vl7H1EmS\npkhaXnW7t9dxtUPSrpKuytbwWS5p7xptpkp6XNLzkn4laYSLLqPPSP2UdGGN3F7Tq3hbIekkSbdL\nWiJpoaQrJL2zRru+zmcz/SxJPo+QdLekxdntZkmTq9r0dS5h5H6WIZeQs/CWdJak3SS9XdJOwBXA\nK8ClhURnZmZmfUPSp4CzgSnA9sDdwMxsP/EymUfaG318dtult+G0bQ3SPu9HAVF9p6QTgC8ChwPv\nB5aS8rpKN4PsgIb9zFzLirk9sDuhdcyuwPeAHYGPACsDv5S02lCDkuRzxH5m+j2fjwInABOAHYDf\nAFdJ2hpKk0sYoZ+Zfs9l7qHmG5P27FwfeAqYDXwg28vTzMzMBtuxwPkRcTGkqxjAx4DPAWf2MrAO\nK9XWqhFxHWlBHiSpRpNjgK9HxM+zNp8BFgIfB2Z0K852NdFPgJf6ObcRsVflz5IOAZ4kFTOzs8N9\nn88m+wn9n89fVB06WdKRpC8c7qUEuYSm+gl9nkvIecU7Ig6MiI0jYrWIeFtEHBQRDxYVnJmZmfUH\nSSuTPvT+euhYRARwPTCxV3EVZGC2VpW0KenqUmVelwC3Ub68AkzKhi7PlzRN0nq9DqhN65Ku7j8D\npc7nCv2sUJp8Shoj6QBgVdKuUqXMZXU/K+7q+1zmXlxN0puBbwN7AqsDfwIOjYi5HY7NWlbvS9zh\njuOUws5tvVBv5NxwZzE115mPY0reYHL4a872rxYSRZLnbTFvHEWeu0jNv64ALsrx2vpsztfV2Mvy\nvmcVZfscbdcuLIpRZgNgLOlqS6WFwJbdD6cwQ1ur/oG0cuyppA/B20TE0h7GVZTxpDeBWnkd3/1w\nCnUtcDnwILAZcAZwjaSJ2ZdIfSW7qv9dYHZEDF01LF0+6/QTSpJPSdsAt5C2iHoe2D8i7pc0kRLl\nsl4/s7tLkctchbekdYHfkr5Z2QNYBGwBPNv50MzMzMxGl9a3VrXRLiIqh+b+XtI9wP3AJOCGngTV\nnmnA1sDOvQ6kYDX7WaJ8zge2A9YBPglcJumDvQ2pEDX7GRF3liWXeVc1PxF4JCIOi4g5EfFwRFzv\n4eZmZmYDbxGwjLT4TaWNgAXdD6c7ImIx0MTWqn1rAWm420DlFSD7fLuIPsytpPOAvYBJEVG5+1Cp\n8tmgn8P0az4j4tWIeCArQL9KGkp+JCXLZYN+1mrbl7nMW3j/PXCHpBnZGPu5kg4rIjAzMzPrHxHx\nCjAH2H3oWDYEdHfg5l7FVTSVfGvV7APuAlbM69qkRY9Km1cASRuTFhTuq9xmxeg+wIci4pHK+8qU\nz0b9rNO+L/NZwxhgbJlyWccY0vSlYfo1l3nneL+D9M3D2cA3ScvWnyvppYj4z04HZ2ZmZn3lHGC6\npDnA7aRVzlcHpvcyqE6SdBZwNWl4+VuA0+jzrVUlrUH68mBoIZd3SNoOeCYiHiXNnz1Z0p+Bh4Cv\nA48BV/Yg3JY16md2m0KaR7oga/dt0miGmcPPNjpJmkbaZmlvYKmkoauhiyPixezvfZ/PkfqZ5boM\n+TydNL/5EWAt4GBgN+AbWZO+zyU07mdZcgn5C+8xwO0R8bXs57uzifBHADkL73uAbXM+fT9yP8ul\n/P28k3zLRvWv8ucyGZR+zgO26XUQAy8iZmR7dk8lDXe8C9ij37eAqVLGrVXfS5onGdnt7Oz4RcDn\nIuJMSasD55NWj54F7BkRL/ci2DY06udRwLuBz5D6+DjpQ/0p2WiOfnEEqW83Vh0/FLgYoCT5HKmf\nyyhHPjckvT7fBCwGfkd6T70BSpNLaNBPSeMoRy5zF95PAPdVHbsP2Lfxw64jLVBXaQmD82HQ/SyP\n8vfzLgal8C5/LpNB6udoKLxvIn3uqVTGha7ri4hppMWOSikiDux1DJ0WEf/FCNMPI+JU0grufauJ\nfk7uVixFiYimppH2ez5H6md2db8M+RxxSm+/5xIa97MsuYT8hfdvGb4lyJak4VYNTCZ9gVGpb0dk\nmZmZ1bFbdqt0P/AvPYjFzMzMRou8i6t9B/iApJMkbSbpIOAw4LzOh2ZmZmZmZmbW/3IV3hFxB/AJ\n0mIG9wBfBY6JiMsKiM3MzMzMzMys7+Udak5EXANc02TzbGL3ohp3vUifrQDfotHez8jZXnWOj/Z+\ndspo72fz+XyszvEX6t43mvq9ao62L9U5Xi+XNXeuqGNZjrZFn7ueTrxm871PPJSrdd7Y6sVSq5/1\n3q86Ye0cbV/7H1W92ImZmZkNCEXkLbxynDwNRf9JYU9gZmbWPw6OiEt6HYSZmZl1X9GF9/rAHqQL\nIC82bm1mZlZK44BNgJl9vuWUmZmZtajQwtvMzMzMzMxs0OVd1dzMzMzMzMzMcnDhbWZmZmZmZlYg\nF95mZmZmZmZmBXLhbWZmZmZmZlYgF95mZmZmZmZmBepJ4S3paEkPSnpB0q2S3teLOIoiaYqk5VW3\ne3sdVzsk7SrpKkl/yfqzd402UyU9Lul5Sb+StHkvYm3HSP2UdGGN3F7Tq3hbIekkSbdLWiJpoaQr\nJL2zRru+zmcz/SxJPo+QdLekxdntZkmTq9r0dS5h5H6WIZdmZmZWXl0vvCV9CjgbmAJsD9wNzJS0\nQbdjKdg8YCNgfHbbpbfhtG0N4C7gKGDYHnSSTgC+CBwOvB9YSsrrKt0MsgMa9jNzLSvm9sDuhNYx\nuwLfA3YEPgKsDPxS0mpDDUqSzxH7men3fD4KnABMAHYAfgNcJWlrKE0uYYR+Zvo9l2ZmZlZSXd/H\nW9KtwG0RcUz2s0gfqM6NiDO7GkxBJE0B9omICb2OpQiSlgMfj4irKo49DpwVEd/Jfl4bWAh8NiJm\n9CbS9tTp54XAOhGxb+8i66zsS68ngd0iYnZ2rIz5rNXP0uUTQNLTwJcj4sIy5nJIVT9LmUszMzMr\nh65e8Za0MulKxa+HjkWq/K8HJnYzli7YIhuufL+kH0t6a68DKoqkTUlXlyrzugS4jfLlFWBSNnR5\nvqRpktbrdUBtWpd0df8ZKHU+V+hnhdLkU9IYSQcAqwI3lTWX1f2suKs0uTQzM7NyWanLz7cBMJZ0\ntaXSQmDLLsdSpFuBQ4A/AG8CTiV9CN4mIpb2MK6ijCcVNLXyOr774RTqWuBy4EFgM+AM4BpJE6Pb\nw0c6IBtx8l1gdkQMrUNQunzW6SeUJJ+StgFuAcYBzwP7R8T9kiZSolzW62d2dylyaWZmZuXU7cJ7\nIETEzIof50m6HXgY2B+4sDdRWSdUDc39vaR7gPuBScANPQmqPdOArYGdex1IwWr2s0T5nA9sB6wD\nfBK4TNIHextSIWr2MyLuLFEuzczMrIS6vbjaImAZafGbShsBC7ocS9dExGLgj0DfrSTcpAWAGLC8\nAkTEg6TXdd/lVtJ5wF7ApIh4ouKuUuWzQT+H6dd8RsSrEfFAVoB+lTSU/EhKlssG/azVti9zaWZm\nZuXU1cI7Il4B5gC7Dx3LhoDuDtzczVi6SdKapA9/DT/096vsA+4CVszr2qTVpEubVwBJGwPr02e5\nzYrRfYAPRcQjlfeVKZ+N+lmnfV/ms4YxwNgy5bKOMaTpS8OUKJdmZmZWAr0Yan4OMF3SHOB24Fhg\ndWB6D2IphKSzgKtJw8vfApwGvAJc2su42iFpDdKXB8oOvUPSdsAzEfEoaf7syZL+DDwEfB14DLiy\nB+G2rFE/s9sU0jzSBVm7b5NGM8wcfrbRSdI00jZLewNLJQ1dDV0cES9mf+/7fI7UzyzXZcjn6aT5\nzY8AawEHA7sB38ia9H0uoXE/y5JLMzMzK6+uF94RMSPb1mcqabjjXcAeEfFUt2Mp0MbAJaSrLU8B\ns4EPRMTTPY2qPe8lzZOM7HZ2dvwi4HMRcaak1YHzSatHzwL2jIiXexFsGxr18yjg3cBnSH18nPSh\n/pRsNEe/OILUtxurjh8KXAxQknyO1M9llCOfG5Jen28CFgO/I72n3gClySU06KekcZQjl2ZmZlZS\nXd/H28zMzMzMzGyQdHtxNTMzMzMzM7OB4sLbzMzMzMzMrEAuvM3MzMzMzMwK5MLbzMzMzMzMrEAu\nvM3MzMzMzMwK5MLbzMzMzMzMrEAuvM3MzMzMzMwK5MLbzMzMzMzMrEAuvM3MzMzMzMwK5MLbzMzM\nzMzMrEAuvM3MzMzMzMwK9P8D6874g8NRfmsAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "fig, ((ax1,ax2),(ax3,ax5),(ax4,ax6),) = plt.subplots(nrows=3, ncols=2)\n", + "plt.rcParams['savefig.facecolor'] = \"0.8\"\n", + "fs = 12 # font size\n", + "fig.set_figwidth(10)\n", + "fig.set_figheight(5)\n", + "\n", + "ax1.imshow(_X.T - _y.T, interpolation='none') ; ax1.set_title('input ($X$) and target ($y$)')\n", + "ax2.imshow(_y_hat[0,:,:].T, interpolation='none') ; ax2.set_title('prediction ($\\hat y$)')\n", + "\n", + "ax3.imshow(_w_w[0,:,:].T, interpolation='none') ; ax3.set_title('write weighting ($w_w$)')\n", + "ax4.imshow(_w_r[0,:,:,0].T, interpolation='none') ; ax4.set_title('read weighting ($w_r$)')\n", + "\n", + "ax5.imshow(_f[0,:,:].T, interpolation='none') ; ax5.set_title('free gate ($f$)') ; ax5.set_aspect(3)\n", + "ax6.imshow(_g_a[0,:,:].T, interpolation='none') ; ax6.set_title('allocation gate ($g_a$)') ; ax6.set_aspect(3)\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/repeat-copy/repeat-copy-rnn.ipynb b/repeat-copy/repeat-copy-rnn.ipynb new file mode 100644 index 0000000..a6cca09 --- /dev/null +++ b/repeat-copy/repeat-copy-rnn.ipynb @@ -0,0 +1,405 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Differentiable Neural Computer (DNC) using a RNN Controller\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from rnn_controller import RNNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 10000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"nn_models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAADyNJREFUeJzt3W2sHNV5wPH/cx24TiAY1U7BKmkgJSmyHdGAQ0oTXhrb\nckRVaKWK4CJVQCIEJP3gqi1BRiKlhKSNiGmhjlq14FQE01RNVKIGjA20KSKUxk5ofA1F4SXQYlMM\nwpGMTY3v6YfZm96X3Xv37J3Z3dn9/6T9sLOzs8+Z5+ycnZ2ZZyKlhCRJqq+RXgcgSZLmx8FckqSa\nczCXJKnmHMwlSao5B3NJkmrOwVySpJpzMJckqeYczCVJqjkHc0mSau5tVS48IhYDa4HngUNVfpYk\nSQNmIXAysDWl9OpsM3Y0mEfEp4HfB04EngB+N6X0701mXQt8rZPPkCRJAFwK3D3bDNmDeUR8ArgF\nuBJ4HFgPbI2I96eU9k2b/XmAi+76dZactnjKC9vWP8iajatmLH/Bys1Z8eRWlh//3mVZ849kxjO+\n4/Km07et386ajauzltVMbin9O1YelfsJWXNf8b23mk5v1d47Vrbf5T65o/myS5O5Mlv1ha0Uv1qn\ny+1rlff9Fn0zV6vc5vfNvM1Pq77Wevk5fb9V8M2zm903S+prrWTntkU8rbbLlceTqby+Vk5+q9su\n7wO+AY2xdDad7JmvB/4ypfS3ABFxFfBrwBXAn06b9xDAktMWs/SME6e8MHr86IxpAAsyg8neoDX5\nzNnknlTQavmji5q3N1f+fXGOzv2ErLmXnnG46fSFLfIL7W9gWy27NLkb2BbTFwJLm0zP7Wv91vdb\nadWX8/tm3g/N/P6Q0/dbBd88u9mxlNTXWsnObYt4Wm2XK48nU3l9rZz8Vr9dnvswdVaOIuIo4Ezg\nwYlpqbjt2nbg7NzoJEnS/OX+4FpCsQPx8rTpL1McP5ckSV3mpWmSJNVc7jHzfcAR4IRp008A9rZ6\n07b1DzJ6/OiUaYt+/rjMj6635euW9TqErlp+yfC0d0WvA+iy4erLw5XdYfreFvopvz8Edk2b1v4V\n3VmDeUrpcETsAFYB9wJERDSe/3mr963ZuKqUk7/qbMW65b0OoauWD1F7+2lz0A3D1ZeHK7srhuqH\nGvRXfj/QeEy2B/irtt7dydnsXwY2Nwb1iUvT3gFs7mBZkiRpnrIH85TS1yNiCXAjxd/rPwDWppRe\nKTs4SZI0t44qwKWUNgGbSo5FkiR1wLPZJUmquU7KuZ4D/AFF8ZilwG+klO4tK6DsqlZHPps1/8jI\nF7Pmj6y5YWTkC1nzj49fl/kJufLW6IYjeZWPPj+SWy62/TV6U+ayrx+vtmJcfl/I62vDJ2+N5ve1\nIZJbYc6+Oavcim790Dc72TM/huI4+TXkj72SJKlknZwAdz9wP/z0sjRJktRDHjOXJKnmHMwlSao5\nB3NJkmquo+vMczWrzb78kmVDWDpQkqRmulibvVPWZpckaTZdrs0eEccAp/L/F4m+NyJOB15LKb2Y\nuzxJkjQ/neyZrwQeprjGPAG3NKZ/FbiipLgkSVKbOrnO/F/wxDlJkvqGg7IkSTWXtWceEdcBvwmc\nBhwEHgWuTSk9Pccbi0cbxsfzaq1X7UifxZNbc29DxfXKc20Y/9/qFt5n9anrXh6x/+pT99Eaze1r\nC/L6WtXbwdw1WXXd7uy+tiCvr1W/Hex938zdMz8HuA34MLAaOAp4ICLeXnZgkiSpPVl75imlCyY/\nj4jLgP+huIPaI+WFJUmS2jXfY+bHU/wD81oJsUiSpA50PJg37ph2K/BISml3eSFJkqQc86kAtwlY\nBnykpFgkSVIHOhrMI+J24ALgnJTSnrnm37Z+O6OLptVmX7eMFeuWd/LxkiQNmC7XZm8M5BcB56WU\nXmjnPWs2rrY2uyRJLXWxNntEbALWARcCByLihMZL+1NK7f+EkCRJpck9Ae4q4Djgn4GXJj0uLjcs\nSZLUrtzrzC3/KklSn3FwliSp5nKPmV8FXA2c3Jg0BtyYUrq/tIhyi49nFvXNraCbX1P5usxPqFbF\nq5PcNZpTU/n63HrKFReQPnIkr172gsy+M2w2HMmr0//5BUdXFEn1IrNvZt83IPN73m/3nMiWMrc7\nufcNyL4HRjX3nNiz8zB3rGxv3tw98xeBa4EzKEq4PgTcGxHLMpcjSZJKknvM/J+mTbo+Iq6muPGK\nVeAkSeqBjivARcQIxVnso8C/lhaRJEnK0knRmBXAd4GFwBvAxSmlH5UdmCRJak8nZ7M/BZwOnAXc\nDtwTER8sNSpJktS27D3zlNJbwLONp9+PiLMoznC/stV7rM0uSVJrY1vGGLtn6qlnh15/s+33z+eu\naRNGgAWzzWBtdkmSWlu+bjnLp+3g7tm5lztW3tnW+3OvM78ZuA94AXgncClwLnBTznIkSVJ5cvfM\nfxb4KrAU2A/8B7A2pfRw2YFJkqT25F5n/qmqApEkSZ2xNrskSTU3rxPgIuKzwM3ArSml32s5Y0pt\nF/2uuhZ6bn3t3Hiqlls7PacWOuTXQ8+vr51ZIzlHZiH68brXp65Ybl3/Dbm19Ossc+Vk10LP/J7n\n3geg3+45kd/XMmuhZ2838+4DUNV2Nme9dLxnHhEforgc7YlOlyFJkuavo8E8Io4F7gI+BbxeakSS\nJClLp3vmfwF8K6X0UJnBSJKkfJ3UZr8E+CWgzbusSpKkKuUWjTkJuBVYnVIaorNdJEnqX7l75mcC\n7wJ2Rvz0PLsFwLkR8RlgNKWZ51tvW/8go8dPq81+yTJWrFvWQciSJA2WXVvGGNsytTb7m/urq82+\nHfjAtGmbgSeBLzYbyAHWbFxlbXZJklpYsW75jJuP7dm5l785s4La7CmlA8CUnw4RcQB4NaX0ZM6y\nJElSOcqoAJd5Ob4kSSrTvG+BmlL6WBmBSJKkzkSLw9zlLDziDGDHJ3dc3v4x89x4cusA5qpw/UD1\nZRXrvDqrjqVyFfed3P/ERt7WXyU8c1W9OnP0Xd/sp5VD/5WLzVX16my3/0w6Zn5mSmnnbPNm/c0e\nETdExPi0x+653ylJkqrSyd/su4BVwMRvi7fKC0eSJOXqZDB/K6X0SumRSJKkjnRyNvv7IuK/I+KZ\niLgrIt5delSSJKltuYP5Y8BlwFrgKuAU4DsRcUzJcUmSpDblFo3ZOunproh4HPgxcDHQskzNtvXb\nGV00tZzrovccx8dvW5vz8bW2a8vuoSpfu2vL2IxqRoNqmNoKw9XesS1jLB+StoLbqV7qdjnXKVJK\n+yPiaeDU2eZbs3H1jEvT/u7Cv5/PR9fO2D3D9SUZ27K7b74kVRumtsJwtXfsnt1DNZi7neqd+ZZz\nnVcFuIg4lmIg3zOf5UiSpM7lXmf+pYg4NyLeExG/AnwTOAxsqSQ6SZI0p9y/2U8C7gYWA68AjwC/\nnFJ6tezAJElSe3JPgFuXufyFAPue3DfjhTf3v8menXubfUjeJ/RT/dFZvPl68/aOZC5+vNk6m0Wv\nVmer/A5iOdfS+nKu3HKuJfW1lu2tWC8qlh5q8b3tu75Z8XYqV9XbtbKUsZ3qRLv9Z9LYuXDOZVZc\nm/23ga9V9gGSJA2+S1NKd882Q9WD+WKKa9KfBw5V9kGSJA2ehcDJwNa5DmdXOphLkqTqzevSNEmS\n1HsO5pIk1ZyDuSRJNedgLklSzfVkMI+IT0fEcxFxMCIei4gP9SKOqkXEDRExPu2xe+539r+IOCci\n7m3cDnc8Ii5sMs+NEfFSRLwREdsiYtYa/v1srvZGxJ1Ncv3tXsU7HxFxXUQ8HhE/iYiXI+KbEfH+\nJvMNRH7bae+g5DciroqIJyJif+PxaER8fNo8A5FXmLu9g5JX6MFgHhGfAG4BbgA+CDwBbI2IJd2O\npUt2AScAJzYeH+1tOKU5BvgBcA1NypdExLXAZ4ArgbOAAxR5PrqbQZZo1vY23MfUXOcWWeoX5wC3\nAR8GVgNHAQ9ExNsnZhiw/M7Z3oZByO+LwLXAGcCZwEPAvRGxDAYurzBHexsGIa+QUurqg+Ke6H82\n6XkA/wX8Ybdj6UJbbwB29jqOLrRzHLhw2rSXgPWTnh8HHAQu7nW8FbX3TuAbvY6tovYuabT5o0OS\n32btHeT8vgpcPuh5bdHegclrV/fMI+Ioil9HD05MS8Ua3Q6c3c1Yuuh9jb9mn4mIuyLi3b0OqGoR\ncQrFL9zJef4J8G8Mbp4Bzm/8TftURGyKiJ/pdUAlOZ7i34jXYCjyO6W9kwxUfiNiJCIuAUaB7wx6\nXqe3d9JLA5HXed3PvANLgAXAy9Omvwz8Ypdj6YbHgMuA/wSWAp+j+NKsSCkd6GFcVTuRYmPYLM8n\nzpx9INwH/APwHPALwBeAb0fE2Y0frLUUEQHcCjySUpo432Ng89uivTBA+Y2IFcB3KaqLvUGx1/1M\nRJzNAOa1VXsbLw9MXrs9mA+VlNLWSU93RcTjwI+Biyn+3tGASCl9fdLTsYj4IfAMcD7wcE+CKscm\nYBnwkV4H0iVN2ztg+X0KOB1YBPwWcE9EnNfbkCrVtL0ppe8PUl67fQLcPuAIxckGk50A9Oa2OV2U\nUtoPPA3U9uzQNu2lOBdiKPMMkFJ6jqK/1zbXEXE7cAFwfkppz6SXBjK/s7R3hjrnN6X0Vkrp2cZg\ntoHib/SrGdC8ztLeZvPWNq9dHcxTSoeBHcCqiWmNv7VWAY92M5ZeiIhjKTrJrBuKumt8IfYyNc/H\nUZwtPPB5BoiIk4DF1DTXjYHtIuBXU0ovTH5tEPM7W3tbzF/r/E4zAiwYxLy2MEJxuHeGOue1F3+z\nfxnYHBE7gMeB9cA7gM09iKVSEfEl4FsUf63/HPBHwGFgSy/jKkNEHEPxw2TizrzvjYjTgddSSi9S\nHHe8PiJ+RHHXvD+muGrhH3sQ7rzN1t7G4waKY297G/P9CcW/MFtnLq2/RcQmistzLgQORMTEntr+\nlNLE3Q8HJr9ztbeR+4HIb0TcTHGc+AXgncClwLnATY1ZBiavMHt7BymvQPcvTWucU3ANRUc5SHFi\nwspen9ZfUTu3UHwRDlJ0pruBU3odV0ltO4/i8p0j0x53TJrncxSXurxB8eU4tddxV9FeihNr7qfY\nIBwCngW+Aryr13F32NZm7TwC/M60+QYiv3O1d5DyC/x1I/6DjfY8AHxsEPM6V3sHKa8pJW+BKklS\n3VmbXZKkmnMwlySp5hzMJUmqOQdzSZJqzsFckqSaczCXJKnmHMwlSao5B3NJkmrOwVySpJpzMJck\nqeYczCVJqjkHc0mSau7/APAgGuehUE1gAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "LSTMStateTuple(c=, h=) LSTMStateTuple(c=, h=)\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n", + "ready to train..." + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(RNNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "for i, (grad, var) in enumerate(gradients):\n", + " if grad is not None:\n", + " gradients[i] = (tf.clip_by_value(grad, -10, 10), var)\n", + " \n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model overview...\n", + "\tvariable \"basic_lstm_cell/weights:0\" has 20480 parameters\n", + "\tvariable \"basic_lstm_cell/biases:0\" has 256 parameters\n", + "\tvariable \"W_z:0\" has 3072 parameters\n", + "\tvariable \"W_v:0\" has 384 parameters\n", + "\tvariable \"W_r:0\" has 60 parameters\n", + "\tvariable \"while_loop/basic_lstm_cell/weights:0\" has 20480 parameters\n", + "\tvariable \"while_loop/basic_lstm_cell/biases:0\" has 256 parameters\n", + "total of 44988 parameters\n" + ] + } + ], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no saved model to load.\n" + ] + } + ], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 0/40000\n", + "\tloss: 0.6937\n", + "Iteration 100/40000\n", + "\tloss: 0.6921\n", + "Iteration 200/40000\n", + "\tloss: 0.4698\n", + "Iteration 300/40000\n", + "\tloss: 0.3195\n", + "Iteration 400/40000\n", + "\tloss: 0.2620\n", + "Iteration 500/40000\n", + "\tloss: 0.2429\n", + "Iteration 600/40000\n", + "\tloss: 0.2198\n", + "Iteration 700/40000\n", + "\tloss: 0.2103\n", + "Iteration 800/40000\n", + "\tloss: 0.1918\n", + "Iteration 831/40000" + ] + } + ], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + " global_step = i\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=global_step)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "fig, ((ax1,ax2),(ax3,ax5),(ax4,ax6),) = plt.subplots(nrows=3, ncols=2)\n", + "plt.rcParams['savefig.facecolor'] = \"0.8\"\n", + "fs = 12 # font size\n", + "fig.set_figwidth(10)\n", + "fig.set_figheight(5)\n", + "\n", + "ax1.imshow(_X.T - _y.T, interpolation='none') ; ax1.set_title('input ($X$) and target ($y$)')\n", + "ax2.imshow(_y_hat[0,:,:].T, interpolation='none') ; ax2.set_title('prediction ($\\hat y$)')\n", + "\n", + "ax3.imshow(_w_w[0,:,:].T, interpolation='none') ; ax3.set_title('write weighting ($w_w$)')\n", + "ax4.imshow(_w_r[0,:,:,0].T, interpolation='none') ; ax4.set_title('read weighting ($w_r$)')\n", + "\n", + "ax5.imshow(_f[0,:,:].T, interpolation='none') ; ax5.set_title('free gate ($f$)') ; ax5.set_aspect(3)\n", + "ax6.imshow(_g_a[0,:,:].T, interpolation='none') ; ax6.set_title('allocation gate ($g_a$)') ; ax6.set_aspect(3)\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/repeat-copy/repeat-copy.ipynb b/repeat-copy/repeat-copy.ipynb deleted file mode 100644 index 11a87a5..0000000 --- a/repeat-copy/repeat-copy.ipynb +++ /dev/null @@ -1,339 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Repeat Copy Task\n", - "### Using a Differentiable Memory Architecture (DNC)\n", - "\n", - "\"DNC\n", - "\n", - "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", - "\n", - "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '../dnc')\n", - "\n", - "from dnc import DNC\n", - "from nn_controller import NNController\n", - "\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Hyperparameters" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "xydim = 6\n", - "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", - "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", - "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", - "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", - "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", - "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", - "tf.app.flags.DEFINE_integer(\"iterations\", 40000, \"Number of training iterations\")\n", - "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", - "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", - "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", - "tf.app.flags.DEFINE_string(\"save_path\", \"models/model.ckpt\", \"Where to save checkpoints\")\n", - "FLAGS = tf.app.flags.FLAGS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Data functions" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAADxVJREFUeJzt3W2sHNV5wPH/c12wEwhGxSlYJQJSkja+jmhsh5QmvDQ2\nMqIqtFJFcKkqXiIEJP3gqi1BIJFSStJGBFqIq6oNOBXh0lRtFFADBgNtigilsRNaX0NReAm02BSD\ncCRju9j39MOsyX3be/fsndndmfv/Sfvh7p6dfc48Z+fc2Zl5JlJKSJKk+hrqdwCSJGlunMwlSao5\nJ3NJkmrOyVySpJpzMpckqeaczCVJqjknc0mSas7JXJKkmnMylySp5n6qyoVHxDHAWuBFYF+VnyVJ\nUsMsAk4ENqWUXp+pYVeTeUR8Bvh94DjgKeB3U0r/Pk3TtcDXu/kMSZIEwEXA3TM1yJ7MI+JTwM3A\n5cCTwHpgU0R8MKW0a1LzFwHOv+vXWPKhJRNeeGj9Zs6+ZU3ux89dZi36oVUbs9qPbblk2uf71t+K\ntVud7fp7x6rOh9xlWw50G1ZnMsfCgjZj4QHgnOkWnxnO2Pcuzmpf1tjM1S63ubd5uGPVYZmfnPcB\nl36v8/HTflxuotgnmajysdknTd1OtTPo/d319C6+9dv3QWsunUk3e+brgb9KKf0tQERcAfwqcCnw\nZ5Pa7gNY8qElLF1x3IQXFi5eOOW5nsidzDMXP9amT33rb8Xarc5FR7frb+cb8KUr3u4uqE7lTuZt\nnl8ELJ1u8ZnhtBs77ZQ1NnO1G8v592w6PLN93gfkjZ9243L67FY+NvukqdupdmrU31kPU2dtDyLi\nMGAl8PCh51Jx27XNwGm50UmSpLnL/ed+CcUOyquTnn+V4vi5JEnqMS9NkySp5nKPme8CDgLHTnr+\nWGBnuzc9tH4zCxcvnPDc4hOOyvzoehtet6zfIfTU8IXzp7/L+x1Aj82vsTy/sju/cjtY/d02Msro\nyPYJz+3fvb/j92dN5imltyNiC7AauBcgIqL191+0e9/Zt6ypy0kGlVm+brjfIfTU8Dzq74f7HUCP\nza+xPL8m8/mV28Hq7/J1w1Pi2bF1J19deWdH7+/mbPYvAxtbk/qhS9PeDWzsYlmSJGmOsifzlNI3\nImIJcAPFz+s/ANamlF4rOzhJkjS7rirApZQ2ABtKjkWSJHXBs9klSaq5SJllmyLidOAPKIrHLAV+\nPaV0b5u2K4Atl225pL4nwOWWtYqoJo6GyFmdA7cq80uc5cntcM3H5iCtzspXZW7lyaEv5i0/M56x\ng5/Laj+0IC+esbFrstrnys3Xnwxllg7OXJ/XHsyrCNh5PDuAvwZYmVLaOlPLbvbMj6A4Tn4V+RUr\nJUlSybo5Ae4BintLHLosTZIk9ZHHzCVJqjknc0mSas7JXJKkmuvqOvNc09VmH163bKBK6UmS1D/b\nWo/xZr2N+Tt6Mplbm12SpJksZ+q9AN65NG1W2ZN5RBwBnMxPrsR7f0ScAryRUno5d3mSJGluutkz\nXwU8SnGNeQJubj3/NeDSkuKSJEkd6uY683/BE+ckSRoYTsqSJNVc1p55RFwD/AbwC8Be4HHg6pTS\ns6VFlFvDOLdmcG5N4qprJA9aDeMFeTWMq6tJTPa6vO7g/2W1r3zsDNryM8fawI2dCpefu+zrxvJi\nzy3mnjsWFmSOharHZtVya4/mj7XDM9tnjs0Ox8+OrQe4Y1Vny8zdMz8duA34GLAGOAx4MCLelbkc\nSZJUkqw985TSueP/joiLgf+luIPaY+WFJUmSOjXXY+ZHU5zR/kYJsUiSpC50PZm37ph2K/BYSml7\neSFJkqQcc6kAtwFYBny8pFgkSVIXuprMI+J24Fzg9JTSjtnaW5tdkqT2RkdGGb1n4o/c+97c3/H7\nuynnejtwPnBmSumlTt5jbXZJktobXjfM8KQd3B1bd3LHqjs7en/udeYbgHXAecCeiDi29dLulFLn\nt3eRJEmlyT0B7grgKOCfgVfGPS4oNyxJktSp3OvMLf8qSdKAcXKWJKnmco+ZXwFcCZzYemoUuCGl\n9EBpEVVcw7jq5efWPK5a1TWMq1x+br3jbJm1x7PHzoGK7wMwaDLXZ/bYOVDhfQAGzVDeyjk4VnHt\n9NxkDZjI3G29dizvPg+5Ol2dOas9d8/8ZeBqYAVFCddHgHsjYlnmciRJUklyj5n/06SnrouIKylu\nvGIVOEmS+qDrCnARMURxFvtC4F9Li0iSJGXppmjMcuC7wCLgLeCClNIPyw5MkiR1ppuz2Z8BTgFO\nBW4H7omIj5QalSRJ6lj2nnlK6QDwfOvP70fEqRRnuF/e7j3WZpckqb1tI6OMjkw89Wz/7gprs09j\nCFgwUwNrs0uS1N7ydcNTdnB3bN3JV1dWU5v9JuB+4CXgPcBFwBnAjTnLkSRJ5cndM/8Z4GvAUmA3\n8B/A2pTSo2UHJkmSOpN7nfmnqwpEkiR1x9rskiTV3JxOgIuIzwE3AbemlH6vbcOUikcHcmubZ9dO\njy9ktSezJPFYxTWSO1yN78itb55bm/3GyKx/nbE+rx3Li6XyuvtDeWMn+74Bmaoea1XX9b8xs3Z6\n7vKzlp071jK/iJXX3c/eTl1TTRxdyt6uVV13P3N9Xpc7firQ9Z55RHyU4nK0p8oLR5Ik5epqMo+I\nI4G7gE8Db5YakSRJytLtnvlXgPtSSo+UGYwkScrXTW32C4FfBFaVH44kScqVWzTmeOBWYE1Kqf9H\n/CVJUvae+UrgvcDWiHfOdV0AnBERnwUWpjT1vMSH1j/MwqMn1Wa/cBnL1y3rImRJkpql17XZNwMf\nnvTcRuBp4IvTTeQAZ9+y2trskiS10dPa7CmlPcCEfx0iYg/wekrp6ZxlSZKkcpRRAS7zcn9JklSm\nOd8CNaX0yTICkSRJ3Yk2h7nLWXjECmDLZVsu6fyYeW48uTUnK+xvN7LL12aWYazz6syNpXJ1XplU\nP9ZyDdLqrHysDdh2p+qxOWjbtapVNX7GHTNfmVLaOlPbrJ/ZI+L6iBib9Ng++zslSVJVuvmZfRuw\nmp+Uoj9QXjiSJClXN5P5gZTSa6VHIkmSutLN2ewfiIj/iYjnIuKuiHhf6VFJkqSO5U7mTwAXA2uB\nK4CTgO9ExBElxyVJkjqUWzRm07g/t0XEk8CPgAuAtmVqHlq/mYWLJ5ZzXXzCUZxz29qcj6+1bSPb\n51X52tGRUYYnVTNqqvmW220jo1MqVTXVfOorzL/+DtJ2qtflXCdIKe2OiGeBk2dqd/Yta6ZcmvZ3\n5/39XD66dkbvmV8b/NF7tg/Ml6Rq8y63I9vnzQZ/PvUV5mF/B2g7NddyrnOqABcRR1JM5DvmshxJ\nktS93OvMvxQRZ0TECRHxy8A3gbeBkUqikyRJs8r9mf144G7gGOA14DHgl1JKr5cdmCRJ6kzuCXDr\nMpe/CGDX07umvLB/9352bN053YfkfcKAlcxsZ/+b0/d3KDOcsenW2Qz6tTr3telvlbFUrk1n2+V2\n0MZmWWOt7Xc30yB91dstu6y+1qX+aFnb5UHbrrVTxnYKqttWjZs7F80aQ8W12X8L+HplHyBJUvNd\nlFK6e6YGVU/mx1Bck/4isK+yD5IkqXkWAScCm2Y7nF3pZC5Jkqo3p0vTJElS/zmZS5JUc07mkiTV\nnJO5JEk115fJPCI+ExEvRMTeiHgiIj7ajziqFhHXR8TYpMf22d85+CLi9Ii4t3U73LGIOG+aNjdE\nxCsR8VZEPBQRM9bwH2Sz9Tci7pwm19/uV7xzERHXRMSTEfHjiHg1Ir4ZER+cpl0j8ttJf5uS34i4\nIiKeiojdrcfjEXHOpDaNyCvM3t+m5BX6MJlHxKeAm4HrgY8ATwGbImJJr2PpkW3AscBxrccn+htO\naY4AfgBcBUy5JCIirgY+C1wOnArsocjz4b0MskQz9rflfibmOrfI0qA4HbgN+BiwBjgMeDAi3nWo\nQcPyO2t/W5qQ35eBq4EVwErgEeDeiFgGjcsrzNLflibkFVJKPX1Q3BP9z8f9HcB/A3/Y61h60Nfr\nga39jqMH/RwDzpv03CvA+nF/HwXsBS7od7wV9fdO4B/7HVtF/V3S6vMn5kl+p+tvk/P7OnBJ0/Pa\npr+NyWtP98wj4jCK/44ePvRcKtboZuC0XsbSQx9o/TT7XETcFRHv63dAVYuIkyj+wx2f5x8D/0Zz\n8wxwVutn2mciYkNE/HS/AyrJ0RS/RrwB8yK/E/o7TqPyGxFDEXEhsBD4TtPzOrm/415qRF7ndD/z\nLiwBFgCvTnr+VeDnexxLLzwBXAz8F7AU+DzFl2Z5SmlPH+Oq2nEUG8Pp8nzc1OaNcD/wD8ALwM8B\nXwC+HRGntf5hraWICOBW4LGU0qHzPRqb3zb9hQblNyKWA9+lqC72FsVe93MRcRoNzGu7/rZebkxe\nez2ZzysppU3j/twWEU8CPwIuoPh5Rw2RUvrGuD9HI+I/geeAs4BH+xJUOTYAy4CP9zuQHpm2vw3L\n7zPAKcBi4DeBeyLizP6GVKlp+5tS+n6T8trrE+B2AQcpTjYY71ighFsTDbaU0m7gWaC2Z4d2aCfF\nuRDzMs8AKaUXKMZ7bXMdEbcD5wJnpZR2jHupkfmdob9T1Dm/KaUDKaXnW5PZtRQ/o19JQ/M6Q3+n\na1vbvPZ0Mk8pvQ1sAVYfeq71s9Zq4PFextIPEXEkxSCZcUNRd60vxE4m5vkoirOFG59ngIg4HjiG\nmua6NbGdD/xKSuml8a81Mb8z9bdN+1rnd5IhYEET89rGEMXh3inqnNd+/Mz+ZWBjRGwBngTWA+8G\nNvYhlkpFxJeA+yh+Wv9Z4I+At4GRfsZVhog4guIfk0N38n1/RJwCvJFSepniuON1EfFDirvm/THF\nVQvf6kO4czZTf1uP6ymOve1stftTil9hNk1d2mCLiA0Ul+ecB+yJiEN7artTSofuftiY/M7W31bu\nG5HfiLiJ4jjxS8B7gIuAM4AbW00ak1eYub9NyivQ+0vTWucUXEUxUPZSnJiwqt+n9VfUzxGKL8Je\nisF0N3BSv+MqqW9nUly+c3DS445xbT5PcanLWxRfjpP7HXcV/aU4seYBig3CPuB54C+B9/Y77i77\nOl0/DwK/M6ldI/I7W3+blF/gb1rx723150Hgk03M62z9bVJeU0reAlWSpLqzNrskSTXnZC5JUs05\nmUuSVHNO5pIk1ZyTuSRJNedkLklSzTmZS5JUc07mkiTVnJO5JEk152QuSVLNOZlLklRzTuaSJNXc\n/wNYM0Jj7W43GwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def get_sequence(length, reps, dim):\n", - " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", - " X = np.vstack(X) ; X[:,dim-1] = 0\n", - " \n", - " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", - " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", - " markers = range(length+1, X.shape[0], 2*length+3)\n", - " X[markers,dim-1] = 1\n", - " return X, y\n", - " \n", - "def next_batch(batch_size, length, reps, dim):\n", - " X_batch = []\n", - " y_batch = []\n", - " for _ in range(batch_size):\n", - " X, y = get_sequence(length, reps, dim)\n", - " X_batch.append(X) ; y_batch.append(y)\n", - " return [X_batch, y_batch]\n", - "\n", - "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", - "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Helper functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def binary_cross_entropy(y_hat, y):\n", - " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", - "\n", - "def llprint(message):\n", - " sys.stdout.write(message)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Build graph, initialize everything" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "building graph...\n", - "defining loss...\n", - "computing gradients...\n", - "init variables... \n" - ] - } - ], - "source": [ - "sess = tf.InteractiveSession()\n", - "\n", - "llprint(\"building graph...\\n\")\n", - "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", - "dnc = DNC(NNController, FLAGS)\n", - "\n", - "llprint(\"defining loss...\\n\")\n", - "y_hat, outputs = dnc.get_outputs()\n", - "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", - "loss = binary_cross_entropy(y_hat, dnc.y)\n", - "\n", - "llprint(\"computing gradients...\\n\")\n", - "gradients = optimizer.compute_gradients(loss)\n", - "grad_op = optimizer.apply_gradients(gradients)\n", - "\n", - "llprint(\"init variables... \\n\")\n", - "sess.run(tf.global_variables_initializer())\n", - "llprint(\"ready to train...\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# tf parameter overview\n", - "total_parameters = 0 ; print \"model overview...\"\n", - "for variable in tf.trainable_variables():\n", - " shape = variable.get_shape()\n", - " variable_parameters = 1\n", - " for dim in shape:\n", - " variable_parameters *= dim.value\n", - " print '\\tvariable \"{}\" has {} parameters' \\\n", - " .format(variable.name, variable_parameters)\n", - " total_parameters += variable_parameters\n", - "print \"total of {} parameters\".format(total_parameters)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "global_step = 0\n", - "saver = tf.train.Saver(tf.global_variables())\n", - "load_was_success = True # yes, I'm being optimistic\n", - "try:\n", - " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", - " ckpt = tf.train.get_checkpoint_state(save_dir)\n", - " load_path = ckpt.model_checkpoint_path\n", - " saver.restore(sess, load_path)\n", - "except:\n", - " print \"no saved model to load.\"\n", - " load_was_success = False\n", - "else:\n", - " print \"loaded model: {}\".format(load_path)\n", - " saver = tf.train.Saver(tf.global_variables())\n", - " global_step = int(load_path.split('-')[-1]) + 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Train loop" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "loss_history = []\n", - "for i in xrange(global_step, FLAGS.iterations + 1):\n", - " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", - "\n", - " rlen = np.random.randint(1, FLAGS.length + 1)\n", - " rreps = np.random.randint(1, FLAGS.reps + 1)\n", - " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", - " tsteps = rreps*(2*rlen+3)\n", - "\n", - " fetch = [loss, grad_op]\n", - " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", - "\n", - " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", - " loss_history.append(step_loss)\n", - "\n", - " if i % 100 == 0:\n", - " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", - " loss_history = []\n", - " if i % FLAGS.save_every == 0 and i is not 0:\n", - " llprint(\"\\n\\tSAVING MODEL\\n\")\n", - " saver.save(sess, FLAGS.save_path, global_step=i)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", - "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", - "\n", - "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", - "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", - "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", - "_y = y[0] ; _X = X[0]\n", - "\n", - "plt.figure(figsize=[8,8])\n", - "\n", - "plt.subplot(611) ; plt.title('y')\n", - "plt.imshow(_y.T, interpolation='none')\n", - "plt.subplot(612) ; plt.title('y_hat')\n", - "plt.imshow(_y_hat[0,:,:].T, interpolation='none')\n", - "plt.subplot(613) ; plt.title('w_W')\n", - "plt.imshow(_w_w[0,:,:].T, interpolation='none')\n", - "plt.subplot(614) ; plt.title('w_r')\n", - "plt.imshow(_w_r[0,:,:,0].T, interpolation='none')\n", - "plt.subplot(615) ; plt.title('free gate')\n", - "plt.imshow(_f[0,:,:].T, interpolation='none')\n", - "plt.subplot(616) ; plt.title('alloc. gate')\n", - "plt.imshow(_g_a[0,:,:].T, interpolation='none')\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/repeat-copy/rnn_controller.py b/repeat-copy/rnn_controller.py index f35b788..e781fdd 100755 --- a/repeat-copy/rnn_controller.py +++ b/repeat-copy/rnn_controller.py @@ -1,7 +1,7 @@ import numpy as np import tensorflow as tf from controller import Controller - +from tensorflow.contrib.rnn.python.ops.core_rnn_cell import LSTMStateTuple """ A 1-Layer recurrent neural network (LSTM) with 64 hidden nodes @@ -10,17 +10,17 @@ A 1-Layer recurrent neural network (LSTM) with 64 hidden nodes class RNNController(Controller): def init_controller_params(self): - rnn_dim = 64 - init = tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32) - - self.params['cell'] = tf.nn.rnn_cell.BasicLSTMCell(rnn_dim, initializer = init) - self.params['state'] = tf.Variable(tf.zeros([self.batch_size, rnn_dim]), trainable=False) - self.params['output'] = tf.Variable(tf.zeros([self.batch_size, rnn_dim]), trainable=False) - + self.rnn_dim = 64 + self.lstm_cell = tf.contrib.rnn.core_rnn_cell.BasicLSTMCell(self.rnn_dim) + self.state = tf.Variable(tf.zeros([self.batch_size, self.rnn_dim]), trainable=False) + self.output = tf.Variable(tf.zeros([self.batch_size, self.rnn_dim]), trainable=False) def nn_step(self, X, state): X = tf.convert_to_tensor(X) - return self.params['cell'](X, state) + return self.lstm_cell(X, state) - def zero_state(self): - return (self.params['output'], self.params['state']) + def update_state(self, update): + return tf.group(self.output.assign(update[0]), self.state.assign(update[1])) + + def get_state(self): + return LSTMStateTuple(self.output, self.state) diff --git a/repeat-copy/rnn_controller.pyc b/repeat-copy/rnn_controller.pyc new file mode 100644 index 0000000..514c33a Binary files /dev/null and b/repeat-copy/rnn_controller.pyc differ diff --git a/repeat-copy/rnn_models/checkpoint b/repeat-copy/rnn_models/checkpoint new file mode 100644 index 0000000..d87012d --- /dev/null +++ b/repeat-copy/rnn_models/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "model.ckpt-1000" +all_model_checkpoint_paths: "model.ckpt-1000" diff --git a/repeat-copy/rnn_models/model.ckpt-1000.data-00000-of-00001 b/repeat-copy/rnn_models/model.ckpt-1000.data-00000-of-00001 new file mode 100644 index 0000000..04457d0 Binary files /dev/null and b/repeat-copy/rnn_models/model.ckpt-1000.data-00000-of-00001 differ diff --git a/repeat-copy/rnn_models/model.ckpt-1000.index b/repeat-copy/rnn_models/model.ckpt-1000.index new file mode 100644 index 0000000..b65497b Binary files /dev/null and b/repeat-copy/rnn_models/model.ckpt-1000.index differ diff --git a/repeat-copy/models/model.ckpt-1000.meta b/repeat-copy/rnn_models/model.ckpt-1000.meta similarity index 82% rename from repeat-copy/models/model.ckpt-1000.meta rename to repeat-copy/rnn_models/model.ckpt-1000.meta index 77dbcc0..cc14ddd 100644 Binary files a/repeat-copy/models/model.ckpt-1000.meta and b/repeat-copy/rnn_models/model.ckpt-1000.meta differ diff --git a/repeat-copy/static/dnc_schema.png b/static/dnc_schema.png similarity index 100% rename from repeat-copy/static/dnc_schema.png rename to static/dnc_schema.png diff --git a/static/repeat_copy_results.png b/static/repeat_copy_results.png new file mode 100644 index 0000000..ae1b453 Binary files /dev/null and b/static/repeat_copy_results.png differ