diff --git a/copy/.ipynb_checkpoints/copy-checkpoint.ipynb b/copy/.ipynb_checkpoints/copy-checkpoint.ipynb deleted file mode 100644 index 566f0fa..0000000 --- a/copy/.ipynb_checkpoints/copy-checkpoint.ipynb +++ /dev/null @@ -1,255 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Differentiable Neural Computer\n", - "\n", - "\"DNC\n", - "\n", - "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", - "\n", - "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '../dnc')\n", - "\n", - "from dnc import DNC\n", - "from nn_controller import NNController" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "length = 10\n", - "xydim = 6\n", - "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", - "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", - "tf.app.flags.DEFINE_integer(\"max_sequence_length\", 2*length+1, \"Maximum sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"length\", length, \"Maximum sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"batch_size\", 2, \"Size of batch in minibatch gradient descent\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", - "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", - "tf.app.flags.DEFINE_integer(\"N\", 15, \"Number of words the DNC memory can store\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", - "tf.app.flags.DEFINE_integer(\"iterations\", 100000, \"Number of training iterations\")\n", - "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", - "tf.app.flags.DEFINE_float(\"momentum\", .9, \"Momentum for RMSProp\")\n", - "tf.app.flags.DEFINE_integer(\"save_every\", 0, \"Save model after this number of train steps\")\n", - "tf.app.flags.DEFINE_string(\"save_dir\", \"models\", \"Directory in which to save checkpoints\")\n", - "tf.app.flags.DEFINE_string(\"log_dir\", \"logs\", \"Directory in which to save logs\")\n", - "FLAGS = tf.app.flags.FLAGS" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def generate_data(batch_size, length, dim):\n", - " X, y = np.zeros((batch_size, 2 * length + 1, dim)), np.zeros((batch_size, 2 * length + 1, dim))\n", - " sequence = np.random.binomial(1, 0.5, (batch_size, length, dim - 1))\n", - "\n", - " X[:, :length, :dim - 1] = sequence\n", - " X[:, length, -1] = 1 # end symbol\n", - " y[:, length + 1:, :dim - 1] = sequence\n", - " \n", - " return X, y\n", - "\n", - "def binary_cross_entropy(y_hat, y):\n", - " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", - "\n", - "def llprint(message):\n", - " sys.stdout.write(message)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "building graph...\n", - "computing gradients...\n", - "init variables... \n", - "starting to train...\n", - "\n", - "Iteration 0/100000\n", - "\tloss: 0.6899\n", - "Iteration 100/100000\n", - "\tloss: 0.6738\n", - "Iteration 200/100000\n", - "\tloss: 0.4000\n", - "Iteration 300/100000\n", - "\tloss: 0.2642\n", - "Iteration 400/100000\n", - "\tloss: 0.2544\n", - "Iteration 500/100000\n", - "\tloss: 0.2533\n", - "Iteration 600/100000\n", - "\tloss: 0.2539\n", - "Iteration 700/100000\n", - "\tloss: 0.2570\n", - "Iteration 800/100000\n", - "\tloss: 0.2507\n", - "Iteration 900/100000\n", - "\tloss: 0.2462\n", - "Iteration 1000/100000\n", - "\tloss: 0.2464\n", - "Iteration 1100/100000\n", - "\tloss: 0.2491\n", - "Iteration 1200/100000\n", - "\tloss: 0.2412\n", - "Iteration 1300/100000\n", - "\tloss: 0.2340\n", - "Iteration 1400/100000\n", - "\tloss: 0.2343\n", - "Iteration 1500/100000\n", - "\tloss: 0.2303\n", - "Iteration 1600/100000\n", - "\tloss: 0.2196\n", - "Iteration 1700/100000\n", - "\tloss: 0.2305\n", - "Iteration 1800/100000\n", - "\tloss: 0.2237\n", - "Iteration 1900/100000\n", - "\tloss: 0.2082\n", - "Iteration 2000/100000\n", - "\tloss: 0.2180\n", - "Iteration 2100/100000\n", - "\tloss: 0.2105\n", - "Iteration 2200/100000\n", - "\tloss: 0.1964\n", - "Iteration 2300/100000\n", - "\tloss: 0.1891\n", - "Iteration 2400/100000\n", - "\tloss: 0.1780\n", - "Iteration 2500/100000\n", - "\tloss: 0.0984\n", - "Iteration 2600/100000\n", - "\tloss: 0.0283\n", - "Iteration 2700/100000\n", - "\tloss: 0.0027\n", - "Iteration 2800/100000\n", - "\tloss: 0.0000\n", - "Iteration 2822/100000" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtsteps\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mrandom_length\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0mstep_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0mloss_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "graph = tf.Graph()\n", - "with graph.as_default():\n", - " with tf.Session(graph=graph) as session:\n", - " llprint(\"building graph...\\n\")\n", - " optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", - " dnc = DNC(NNController, FLAGS)\n", - "\n", - " # define loss\n", - " y_hat, _ = dnc.get_outputs()\n", - " y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", - " loss = binary_cross_entropy(y_hat, dnc.y)\n", - " \n", - " llprint(\"computing gradients...\\n\")\n", - " gradients = optimizer.compute_gradients(loss)\n", - " grad_op = optimizer.apply_gradients(gradients)\n", - " \n", - " llprint(\"init variables... \\n\")\n", - " session.run(tf.global_variables_initializer())\n", - " llprint(\"starting to train...\\n\\n\")\n", - "\n", - " loss_history = []\n", - "\n", - " for i in xrange(FLAGS.iterations + 1):\n", - " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", - "\n", - " random_length = np.random.randint(1, FLAGS.length + 1)\n", - " X, y = generate_data(FLAGS.batch_size, random_length, FLAGS.xlen)\n", - "\n", - " fetch = [loss, grad_op]\n", - " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: 2 * random_length + 1}\n", - " \n", - " step_loss, _ = session.run(fetch, feed_dict=feed)\n", - "\n", - " loss_history.append(step_loss)\n", - "\n", - " if i % 100 == 0:\n", - " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", - " loss_history = []" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/copy/copy.ipynb b/copy/copy.ipynb deleted file mode 100644 index 566f0fa..0000000 --- a/copy/copy.ipynb +++ /dev/null @@ -1,255 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Differentiable Neural Computer\n", - "\n", - "\"DNC\n", - "\n", - "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", - "\n", - "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '../dnc')\n", - "\n", - "from dnc import DNC\n", - "from nn_controller import NNController" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "length = 10\n", - "xydim = 6\n", - "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", - "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", - "tf.app.flags.DEFINE_integer(\"max_sequence_length\", 2*length+1, \"Maximum sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"length\", length, \"Maximum sequence length\")\n", - "tf.app.flags.DEFINE_integer(\"batch_size\", 2, \"Size of batch in minibatch gradient descent\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", - "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", - "tf.app.flags.DEFINE_integer(\"N\", 15, \"Number of words the DNC memory can store\")\n", - "\n", - "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", - "tf.app.flags.DEFINE_integer(\"iterations\", 100000, \"Number of training iterations\")\n", - "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", - "tf.app.flags.DEFINE_float(\"momentum\", .9, \"Momentum for RMSProp\")\n", - "tf.app.flags.DEFINE_integer(\"save_every\", 0, \"Save model after this number of train steps\")\n", - "tf.app.flags.DEFINE_string(\"save_dir\", \"models\", \"Directory in which to save checkpoints\")\n", - "tf.app.flags.DEFINE_string(\"log_dir\", \"logs\", \"Directory in which to save logs\")\n", - "FLAGS = tf.app.flags.FLAGS" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def generate_data(batch_size, length, dim):\n", - " X, y = np.zeros((batch_size, 2 * length + 1, dim)), np.zeros((batch_size, 2 * length + 1, dim))\n", - " sequence = np.random.binomial(1, 0.5, (batch_size, length, dim - 1))\n", - "\n", - " X[:, :length, :dim - 1] = sequence\n", - " X[:, length, -1] = 1 # end symbol\n", - " y[:, length + 1:, :dim - 1] = sequence\n", - " \n", - " return X, y\n", - "\n", - "def binary_cross_entropy(y_hat, y):\n", - " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", - "\n", - "def llprint(message):\n", - " sys.stdout.write(message)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "building graph...\n", - "computing gradients...\n", - "init variables... \n", - "starting to train...\n", - "\n", - "Iteration 0/100000\n", - "\tloss: 0.6899\n", - "Iteration 100/100000\n", - "\tloss: 0.6738\n", - "Iteration 200/100000\n", - "\tloss: 0.4000\n", - "Iteration 300/100000\n", - "\tloss: 0.2642\n", - "Iteration 400/100000\n", - "\tloss: 0.2544\n", - "Iteration 500/100000\n", - "\tloss: 0.2533\n", - "Iteration 600/100000\n", - "\tloss: 0.2539\n", - "Iteration 700/100000\n", - "\tloss: 0.2570\n", - "Iteration 800/100000\n", - "\tloss: 0.2507\n", - "Iteration 900/100000\n", - "\tloss: 0.2462\n", - "Iteration 1000/100000\n", - "\tloss: 0.2464\n", - "Iteration 1100/100000\n", - "\tloss: 0.2491\n", - "Iteration 1200/100000\n", - "\tloss: 0.2412\n", - "Iteration 1300/100000\n", - "\tloss: 0.2340\n", - "Iteration 1400/100000\n", - "\tloss: 0.2343\n", - "Iteration 1500/100000\n", - "\tloss: 0.2303\n", - "Iteration 1600/100000\n", - "\tloss: 0.2196\n", - "Iteration 1700/100000\n", - "\tloss: 0.2305\n", - "Iteration 1800/100000\n", - "\tloss: 0.2237\n", - "Iteration 1900/100000\n", - "\tloss: 0.2082\n", - "Iteration 2000/100000\n", - "\tloss: 0.2180\n", - "Iteration 2100/100000\n", - "\tloss: 0.2105\n", - "Iteration 2200/100000\n", - "\tloss: 0.1964\n", - "Iteration 2300/100000\n", - "\tloss: 0.1891\n", - "Iteration 2400/100000\n", - "\tloss: 0.1780\n", - "Iteration 2500/100000\n", - "\tloss: 0.0984\n", - "Iteration 2600/100000\n", - "\tloss: 0.0283\n", - "Iteration 2700/100000\n", - "\tloss: 0.0027\n", - "Iteration 2800/100000\n", - "\tloss: 0.0000\n", - "Iteration 2822/100000" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtsteps\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mrandom_length\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0mstep_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0mloss_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "graph = tf.Graph()\n", - "with graph.as_default():\n", - " with tf.Session(graph=graph) as session:\n", - " llprint(\"building graph...\\n\")\n", - " optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", - " dnc = DNC(NNController, FLAGS)\n", - "\n", - " # define loss\n", - " y_hat, _ = dnc.get_outputs()\n", - " y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", - " loss = binary_cross_entropy(y_hat, dnc.y)\n", - " \n", - " llprint(\"computing gradients...\\n\")\n", - " gradients = optimizer.compute_gradients(loss)\n", - " grad_op = optimizer.apply_gradients(gradients)\n", - " \n", - " llprint(\"init variables... \\n\")\n", - " session.run(tf.global_variables_initializer())\n", - " llprint(\"starting to train...\\n\\n\")\n", - "\n", - " loss_history = []\n", - "\n", - " for i in xrange(FLAGS.iterations + 1):\n", - " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", - "\n", - " random_length = np.random.randint(1, FLAGS.length + 1)\n", - " X, y = generate_data(FLAGS.batch_size, random_length, FLAGS.xlen)\n", - "\n", - " fetch = [loss, grad_op]\n", - " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: 2 * random_length + 1}\n", - " \n", - " step_loss, _ = session.run(fetch, feed_dict=feed)\n", - "\n", - " loss_history.append(step_loss)\n", - "\n", - " if i % 100 == 0:\n", - " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", - " loss_history = []" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/copy/nn_controller.pyc b/copy/nn_controller.pyc deleted file mode 100644 index e93a810..0000000 Binary files a/copy/nn_controller.pyc and /dev/null differ diff --git a/dnc/dnc.pyc b/dnc/dnc.pyc index f0378d2..03d1dfa 100644 Binary files a/dnc/dnc.pyc and b/dnc/dnc.pyc differ diff --git a/copy/.ipynb_checkpoints/debug-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/debug-checkpoint.ipynb similarity index 100% rename from copy/.ipynb_checkpoints/debug-checkpoint.ipynb rename to repeat-copy/.ipynb_checkpoints/debug-checkpoint.ipynb diff --git a/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb new file mode 100644 index 0000000..41f68e5 --- /dev/null +++ b/repeat-copy/.ipynb_checkpoints/repeat-copy-checkpoint.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Using a Differentiable Memory Architecture (DNC)\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from nn_controller import NNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 40000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAD4lJREFUeJzt3X+MHOV9x/H39y7mnJjYVu3UtkoaSEiK7g7R2A4pTQw0\ntmVEVNNKlcMVqQISIUPSP65qS5CRnFIX0kbEtFBH/WU7FeHcVE1URw0Y29CmiFAXO6G5MxSFH4GW\ns4tBOJLxufbd0z9mj97t7ezud3ZmZ2f285L2j5195tnvM8+z8+zuzHzHQgiIiIhIcfXkHYCIiIi0\nRpO5iIhIwWkyFxERKThN5iIiIgWnyVxERKTgNJmLiIgUnCZzERGRgtNkLiIiUnCazEVERAruXVlW\nbmZLgA3Ay8BElu8lIiJSMvOBC4F9IYQ36hVMNJmb2eeB3wWWA88Avx1C+PcaRTcA30jyHiIiIgLA\nDcBD9Qq4J3Mz+wxwL3ALcAgYBvaZ2UdCCCeqir8McN2Dv8rSS5bMemH/8EHWb187p/6e1btd8Uw9\nfaOrfK+zfq+4TPf7iL7ZVJs6fJOvfmcq/Z2rfV1889PnnPXPi3nlEeAaV11z+Rr72afPusqnNdby\nGsvu+lMaa/uHD7B++7o5y7Mfa9nVH1937U+ud6x59zveO2Z4+zauc1Mby5nv1+L2O7HvELM8pn8P\n+8ZmdvGfAL4Flbm0niS/zIeBvwgh/C2AmW0GPg3cDPxJVdkJgKWXLGHFyuWzXuhb3DdnGfgP4k/V\nqKOeXmf9XnF9Oh9YUWO5N37/fXF8g37FSt9OCs6LWR7XYg9fY1es/F9X+bTGWl5jOev648ba/Jj2\nZj/Wsqw/ru7a49g71rz7Hfdk7uzbuM7Nayz792tx+53Yd4hZHte/vrGZffyND1O7+sjM5gGrgIPT\ny0J027UDwBXe6ERERKR13i9cS4m+ZB6vWn6c6Pi5iIiItJkuTRMRESk47zHzE8AksKxq+TLgWNxK\n+4cP0re4b9ayRT+/0PnWxTaYdwBt1z0tHri+P+8Q2qq72ts94xi6rW+hs/r3R8Bo1bLmr+h2TeYh\nhLNmdhhYC+wFMDOrPP+zuPXWb18bc8JM9+ikIdMel+YdQNsMDnXXDnBgaCDvENqouz653TaWO6t/\nL2XufnMc+Mum1k5yNvtXgd2VSX360rT3ALsT1CUiIiItck/mIYRvmtlS4C6iv9d/CGwIIbyednAi\nIiLSWKIMcCGEHcCOlGMRERGRBHQ2u4iISMElSee6Bvg9ouQxK4BfCyHsbbBS9GjC1OQXXfH09HzZ\nVd7LnYlpyhe/V5Ob8R1bJn2ZjP6ox5sm0av5LbplypshzMe5Kd1jzV1/r6/+rMean6/F/rHmrL+3\n+fqzHmvu/Yh3P2j3+N7AOTg7b6z5tqh3P7jNnGPTu1+eai6D4PiRs+xc3VydSX6ZLyA6Tn4b/jEq\nIiIiKUtyAtwjRHfRmL4sTURERHKkY+YiIiIFp8lcRESk4DSZi4iIFFyi68y99g8foG/R7NzsA0P9\nDHZVWkgREZHaxkbGGNtzdNayibfONL1+Wybz9dvXdX1udhERkTgDQwNz7nswfuQYO1fvamr9JNeZ\nLwAu5v+vrPugmV0GvBlCeNVbn4iIiLQmyS/z1cDjRNeYB+DeyvKvAzenFJeIiIg0Kcl15v+CTpwT\nERHpGJqURURECs71y9zM7gB+HbgEOA08CdweQni+7oohRI8mePNTe016cx5748k4KV6Tm/EdnvzU\nSWyZbC7H8DRPPP5N6VthsuPyTTtlPNbc9wFoMt90J3JvSufnMOs8/e7wvYm4Oy7ZZ3Z5+pPU792g\nzW5Oz2b3/jJfA9wPfBxYB8wDHjWzdzvrERERkZS4fpmHEK6d+dzMbgT+h+gOak+kF5aIiIg0q9Vj\n5ouJ/nB6M4VYREREJIHEk3nljmn3AU+EEI42Ki8iIiLZaCUD3A6gH/hESrGIiIhIAokmczN7ALgW\nWBNCGG9Ufv/wQfoWV+Vmv76fwaH+JG8vIiJSKqMjY4yNzP6T+8zJDHOzVyby64CrQgivNLPO+u1r\nlZtdREQkxuDQwJybj40fOcbfrMogN7uZ7QCGgI3AKTNbVnnpZAhhwlOXiIiIpMN7AtxmYCHwz8Br\nMx6b0g1LREREmuW9zlzpX0VERDqMJmcREZGC8x4z3wzcClxYWTQG3BVCeCStgDJOkQw9vneYKnr+\n7oy3qDm/Dm6ZOutbwcOZH7mnx5t331d8KuP7AExN3eEq7+W+D0CPM/+1N/f7pG/sePJx3+kdl85c\n5e77ADi3fW/G97TIWub3AXDf0+I83wodwPvL/FXgdmAlUQrXx4C9ZqZrzERERHLiPWb+T1WL7jSz\nW4luvKIscCIiIjlInAHOzHqIzmLvA/41tYhERETEJUnSmEHg+8B84G1gUwjhx2kHJiIiIs1Jcjb7\nc8BlwOXAA8AeM/toqlGJiIhI09y/zEMI54AXK09/YGaXE53hfkvcOsrNLiIiEq/tudlr6AF66xVQ\nbnYREZF47c7NfjfwMPAK8F7gBuBKYJunHhEREUmP95f5zwJfB1YAJ4H/ADaEEB5POzARERFpjvc6\n889lFYiIiIgko9zsIiIiBdfSCXBm9kXgbuC+EMLv1CnYdPLdyYzzWbt5kwZnzJ3DeNKXw9iTzzqJ\nDtucLt5c617mzB/d03OPq3zWudy9vLnW3ULzg22bM6/8nc7PVdZ59737TW8u96zHmvs+AM79lDf3\nvn+/6cvlvs2ajb/5KTrxL3Mz+xjR5WjPJK1DREREWpdoMjez84EHgc8Bb6UakYiIiLgk/WX+58B3\nQgiPpRmMiIiI+CXJzX498IvA6vTDERERES9v0pgLgPuAdSGEjM9eERERkWZ4f5mvAt4HHDF757zk\nXuBKM/sC0BfC3PMS9w8foG9RVW72of45qetERES602jlMdNE02t7J/MDwKVVy3YDzwJfrjWRA6zf\nvk652UVERGINVh4zjQN/1dTa3gxwp4BZt3Uxs1PAGyGEZz11iYiISDrSyADnvNxfRERE0tTyLVBD\nCJ9KIxARERFJxmIOc6dTudlK4PBnD9+U3TFzb/xZ5xN1xpN1mkevTtucmeq0xnrj8aZ/fVe2Y63o\nm9PDHXvRN47GWl1ZjbXxI8fYuXoXwKoQwpF6ZV1/s5vZVjObqnocbbymiIiIZCXJ3+yjwFpg+rvO\nufTCEREREa8kk/m5EMLrqUciIiIiiSQ5m/3DZvbfZvaCmT1oZu9PPSoRERFpmncyfwq4EdgAbAYu\nAr5nZgtSjktERESa5E0as2/G01EzOwT8BNgE7Ipbr1Y610UfWMg192/wvH2hjY4cZXCoP+8w2mZ0\nZKxr0vV2U1uhu9o7NjLGQJe0FbSfytPYyBhje2afTz7x1pmm12/pOvMQwkkzex64uF65Wulc/27j\n37fy1oUztqe7PiRjI0c75kOStW5qK3RXe8f2HO2qyVz7qfwMDA3MGWszLk1rqKUMcGZ2PtFEPt5K\nPSIiIpKc9zrzr5jZlWb2ATP7ZeDbwFlgJJPoREREpCHv3+wXAA8BS4DXgSeAXwohvJF2YCIiItIc\n7wlwQ8765wOcePbEnBfOnDzD+JFjzupqBuUrn1NevzNv1W5vjzP8qTS2WR1pbc7U+jdLKTU2t7Hs\nTbGZ0liLa29BPoouEzGf27Kmc43bT+U11uJ02n4qq3SuJ557Z+6c36hs1rnZfxP4RmZvICIiUn43\nhBAeqlcg68l8CdE16S8DE5m9kYiISPnMBy4E9jU6nJ3pZC4iIiLZa+nSNBEREcmfJnMREZGC02Qu\nIiJScJrMRURECi6XydzMPm9mL5nZaTN7ysw+lkccWTOzrWY2VfU42njNzmdma8xsb+V2uFNmtrFG\nmbvM7DUze9vM9ptZ3Rz+naxRe81sV42+/m5e8bbCzO4ws0Nm9lMzO25m3zazj9QoV4r+baa9Zelf\nM9tsZs+Y2cnK40kzu6aqTCn6FRq3tyz9CjlM5mb2GeBeYCvwUeAZYJ+ZLW13LG0yCiwDllcen8w3\nnNQsAH4I3EaNlBJmdjvwBeAW4HLgFFE/n9fOIFNUt70VDzO7r71JljrFGuB+4OPAOmAe8KiZvXu6\nQMn6t2F7K8rQv68CtwMrgVXAY8BeM+uH0vUrNGhvRRn6FUIIbX0Q3RP9T2c8N+C/gN9vdyxtaOtW\n4EjecbShnVPAxqplrwHDM54vBE4Dm/KON6P27gK+lXdsGbV3aaXNn+yS/q3V3jL37xvATWXv15j2\nlqZf2/rL3MzmEX07Oji9LERb9ABwRTtjaaMPV/6afcHMHjSz9+cdUNbM7CKib7gz+/mnwL9R3n4G\nuLryN+1zZrbDzH4m74BSspjo34g3oSv6d1Z7ZyhV/5pZj5ldD/QB3yt7v1a3d8ZLpejXlu5nnsBS\noBc4XrX8OPALbY6lHZ4CbgT+E1gBfInoQzMYQjiVY1xZW060M6zVz8vnFi+Fh4F/AF4CPgTcA3zX\nzK6ofGEtJDMz4D7giRDC9Pkepe3fmPZCifrXzAaB7xNlF3ub6Ff3C2Z2BSXs17j2Vl4uTb+2ezLv\nKiGEfTOejprZIeAnwCaiv3ekJEII35zxdMzMfgS8AFwNPJ5LUOnYAfQDn8g7kDap2d6S9e9zwGXA\nIuA3gD1mdlW+IWWqZntDCD8oU7+2+wS4E8Ak0ckGMy0DOvwWW60LIZwEngcKe3Zok44RnQvRlf0M\nEEJ4iWi8F7avzewB4Frg6hDC+IyXStm/ddo7R5H7N4RwLoTwYmUy20L0N/qtlLRf67S3VtnC9mtb\nJ/MQwlngMLB2elnlb621wJPtjCUPZnY+0SCpu6MousoH4hiz+3kh0dnCpe9nADO7AFhCQfu6MrFd\nB/xKCOGVma+VsX/rtTemfKH7t0oP0FvGfo3RQ3S4d44i92sef7N/FdhtZoeBQ8Aw8B5gdw6xZMrM\nvgJ8h+iv9Z8D/gA4C4zkGVcazGwB0ReT6TsFf9DMLgPeDCG8SnTc8U4z+zHRXfP+kOiqhX/MIdyW\n1Wtv5bGV6NjbsUq5Pyb6F2bf3No6m5ntILo8ZyNwysymf6mdDCFM3/2wNP3bqL2Vvi9F/5rZ3UTH\niV8B3gvcAFwJbKsUKU2/Qv32lqlfgfZfmlY5p+A2ooFymujEhNV5n9afUTtHiD4Ip4kG00PARXnH\nlVLbriK6fGey6rFzRpkvEV3q8jbRh+PivOPOor1EJ9Y8QrRDmABeBL4GvC/vuBO2tVY7J4HfqipX\niv5t1N4y9S/w15X4T1fa8yjwqTL2a6P2lqlfQwi6BaqIiEjRKTe7iIhIwWkyFxERKThN5iIiIgWn\nyVxERKTgNJmLiIgUnCZzERGRgtNkLiIiUnCazEVERApOk7mIiEjBaTIXEREpOE3mIiIiBafJXERE\npOD+D2xMPvufese9AAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n", + "ready to train..." + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(NNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model overview...\n", + "\tvariable \"W1:0\" has 2048 parameters\n", + "\tvariable \"b1:0\" has 128 parameters\n", + "\tvariable \"W2:0\" has 32768 parameters\n", + "\tvariable \"b2:0\" has 256 parameters\n", + "\tvariable \"W_z:0\" has 12288 parameters\n", + "\tvariable \"W_v:0\" has 1536 parameters\n", + "\tvariable \"W_r:0\" has 60 parameters\n", + "total of 49084 parameters\n" + ] + } + ], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loaded model: models/model.ckpt-5000\n" + ] + } + ], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 5100/40000\n", + "\tloss: 0.0911\n", + "Iteration 5200/40000\n", + "\tloss: 0.0515\n", + "Iteration 5300/40000\n", + "\tloss: 0.0327\n", + "Iteration 5400/40000\n", + "\tloss: 0.0205\n", + "Iteration 5500/40000\n", + "\tloss: 0.0244\n", + "Iteration 5600/40000\n", + "\tloss: 0.1585\n", + "Iteration 5700/40000\n", + "\tloss: nan\n", + "Iteration 5800/40000\n", + "\tloss: nan\n", + "Iteration 5900/40000\n", + "\tloss: nan\n", + "Iteration 6000/40000\n", + "\tloss: nan\n", + "\n", + "\tSAVING MODEL\n", + "Iteration 6100/40000\n", + "\tloss: nan\n", + "Iteration 6200/40000\n", + "\tloss: nan\n", + "Iteration 6300/40000\n", + "\tloss: nan\n", + "Iteration 6400/40000\n", + "\tloss: nan\n", + "Iteration 6500/40000\n", + "\tloss: nan\n", + "Iteration 6600/40000\n", + "\tloss: nan\n", + "Iteration 6700/40000\n", + "\tloss: nan\n", + "Iteration 6800/40000\n", + "\tloss: nan\n", + "Iteration 6900/40000\n", + "\tloss: nan\n", + "Iteration 7000/40000\n", + "\tloss: nan\n", + "\n", + "\tSAVING MODEL\n", + "Iteration 7100/40000\n", + "\tloss: nan\n", + "Iteration 7200/40000\n", + "\tloss: nan\n", + "Iteration 7300/40000\n", + "\tloss: nan\n", + "Iteration 7391/40000" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdnc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtsteps\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtsteps\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mstep_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mloss_history\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=i)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python2.7/site-packages/matplotlib/colors.py:938: UserWarning: Warning: converting a masked element to nan.\n", + " vmin = float(vmin)\n", + "/usr/local/lib/python2.7/site-packages/matplotlib/colors.py:939: UserWarning: Warning: converting a masked element to nan.\n", + " vmax = float(vmax)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqcAAAKECAYAAAA63ZOcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3X2YZVV94PvvrwmhFaG4AQMYfAccGAhapSBKNypMmmEy\nbbzxokdMxhBCQOU69dxcX0YHFL1OEh9tRkknJrkjYMvh6jWM7VVoo6CAiGgVokgTH0Fe5E1eYpE0\nkoD9u3/sXdSp6nrr6qqzdtX+fp5nP91nnb1P/dY6q8751dp7rR2ZiSRJktQEq0oHIEmSJI0zOZUk\nSVJjmJxKkiSpMUxOJUmS1Bgmp5IkSWoMk1NJkiQ1hsmpJEmSGsPkVJIkSY1hcipJkqTGMDmVJElS\nY5icSpIkqTFMTiVJktQYJqeSJElqDJNTSWqAiHhVRGyPiNdO89yb6ueOKRGbJPVTZGbpGCRJQETc\nCXw7M0+ZUv4l4JDMPLRMZJLUP46cSlJzbAJ+OyL2Gi+IiP2Afwd8ulhUktRHJqeS1BwXA6uB1/eU\nvRHYDfhMkYgkqc88rS9JDRIR3wb+KTNPrB9fB2RmvrJsZJLUH46cSlKzXAwcHxHPiogXAi/HU/qS\nWsSRU0lqkIjYF7gX+C/A04H3As/KzEeKBiZJfWJyKkkNExH/E3ge1fWnt2bm75SNSJL6x9P6ktQ8\nFwO/CRxCNYNfklrDkVNJapiI2B24HwjggMz818IhSVLf/ErpACRJO9gOPAl8wcRUUtt4Wl+Smud1\nwH5Up/clqVU8rS9JDRERRwNHAe8DfpaZLysckiT1nSOnktQcZwF/QXW96X8qHIskFeHIqSRJkhrD\nkVNJkiQ1hrP1Z1DfpWUdcAfweNloJEmS+mI11U1AtmTmwyUCMDmd2TrgM6WDkCRJKuBU4JISP7h1\nyWlEvA34E+AA4Cbg7Mz8zjS73lH9879SregynSuAkxY9xpVnqdpp4ddL/xF/s6Dj/oYzFvwzZ2df\nmh/bacJs/X8L1d/XOzpjgX1/V+3K7Ia/4Y8WdNxcv+czt9JS/q4vRyvp927hPXG2352lbKGFRrzw\nPvwQ8HfwVB7Uf61KTiPiDcBHgTOAG4BhYEtEHJqZD03ZvT6Vvx9w4AyvuHqW5zRhqdpp4R8yC49m\nqd5v+9L82E4TZuv/M7dTqdbbtam3C4t6rqNm7032swkr6fduab43lrKFSnzT1Ypd0ti2CVHDwCcz\n8+LMvBU4E3gMOK1sWJIkSYIWJaf1vaqHgK+Nl2W1jtZXgWNLxSVJkqQJrUlOqc7P7wY8MKX8Aarr\nTyVJklRYm5LTJXBE6QCWCdtpbrbR/NhO82M7zYetNF+21FxsocXVpglRDwG/BPafUr4/1a0CZ3AF\n1aXOvY4Ajqw3zc12mpttND+20/z4VTkfttJ8+Xs3l+XbQj8Abp5SVn5p99Ykp5n5RESMACcAmwEi\nIurHH5/5yJNYObMUJUmSxk030HYf8NcFYpnQmuS09jHgwjpJHV9K6unAhSWDkiRJUqVVyWlmfjYi\n9gPOozqd/z1gXWY+WDYySZIkQcuSU4DM3AhsLB2HJEmSduRsfUmSJDWGyakkSZIaw+RUkiRJjdG6\na04lgFjgcefwgQX/zPM4Z8HHnsN5Cz52oc7j3L7/TK1cC/2dgzL9X02XCzrKvrQ8tGrkNCLWRMTm\niLgnIrZHxPrSMUmSJGlCq5JTYE+q5aPeykL/7JIkSdKSadVp/cy8gup+pON3h5IkSVKDtG3kVJIk\nSQ1mcipJkqTGMDmVJElSY7TqmtOFuQJYPaXsCODIArFIkiQtlh8AN08pe7xEIJOYnM7pJODA0kFI\nkiQtsiPZcbDtPuCvC8QyoVXJaUTsCRzMxHrQL4iIo4BHMvPucpFJkiQJWpacAi8FrqJa4zSBj9bl\nFwGnlQpKkiRJlVYlp5n5DZwEJkmS1FgmapIkSWoMk1NJkiQ1RqtO60vjcoHHncc5C/6Z53Dego/d\nFbsSs7RYFvo7Bwvvw6V+59QPC7sD+XL8PDx3gf34HD6woOPuA/5mQUcuHkdOJUmS1BitSU4j4j0R\ncUNEPBoRD0TEZRFxaOm4JEmSNKE1ySmwBvgEcAxwIrA78JWIeFrRqCRJkvSU1lxzmpkn9z6OiLcA\nPwOGgGtLxCRJkqTJ2jRyOtU+VNfoP1I6EEmSJFVamZxGRADnA9dm5i2l45EkSVKlNaf1p9gIHA68\nsnQgkiRJmtC65DQiLgBOBtZk5n1zH3EFsHpK2RHAkYsemyRJUr/cXG+9Hi8RyBStSk7rxPS1wPGZ\nedf8jjoJOHAJo5IkSeq/I+qtVxMW4W9NchoRG4EOsB7YFhH710+NZWYT/lCQJElqvTZNiDoT2Bv4\nOnBvz3ZKwZgkSZLUozUjp5nZpkRckiRpWTJhkyRJUmOYnEqSJKkxIjNLx9BIETEIjMAZOFt/JVpY\nvz+H8xb8E2PBR+6ahf6Gn8e5ixqHmmL5febvyu/dQtn/taOF/+7sSh8+j3MWfOzCPDVffygzR/v8\nwwFHTiVJktQgrUlOI+LMiLgpIsbq7bqIOKl0XJIkSZrQmuQUuBt4FzAIDAFXApsj4vCiUUmSJOkp\nbVpK6ktTit4XEWcBxwC3FAhJkiRJU7QmOe0VEauoFt/fA7imcDiSJEmqtSo5jYgjgG8Bq4HHgFMy\n88dlo5IkSdK4ViWnwK3AUcAA8Hrg0og4PjNvnPmQK6hy2V5HAEcuUYiSJEn9cHO99Xq8RCCTtCo5\nzcwngdvrhzdGxNHAWVSLmc7gJFznVJIkrTxH1Fuvp9Y5LaZNs/WnswrYrXQQkiRJqrRm5DQiPgxc\nDtwF7AWcCqwFPlQyLkmSJE1oTXIK/DpwEdU5+jHg+8C6zLyqaFSSJEl6SmuS08w8vXQMkiRJml3b\nrzmVJElSg5icSpIkqTFac1pfK1Eu+MhzOG9Bx53HOQv+mecu8GfCrtR012KWei309wbgPM7dhWMX\n1od3JV413cI+FUv1iV35HF7499VCf+digcctntaOnEbEuyNie0R8rHQskiRJqrQyOY2Il1EtvH9T\n6VgkSZI0oXXJaUQ8A9gEnA78vHA4kiRJ6tG65BT4C+CLmXll6UAkSZI0WasmREXEG4EXAy8tHYsk\nSZJ21JrkNCIOAs4HTszMJ0rHI0mSpB21JjkFhoBnAqMRMb5Owm7A2oh4O7BHZk6zNsUVwOopZUcA\nRy5dpJIkSUvuB8DNU8oeLxHIJG1KTr/KjhnlhcBW4E+nT0wBTgIOXMq4JEmSCjiSHVOj+4C/LhDL\nhNYkp5m5DbiltywitgEPZ+bWMlFJkiSpVxtn6/falRvvSJIkaZG1ZuR0Opn5mtIxSJIkaULbR04l\nSZLUICankiRJagyTU0mSJDVGzLiCUstFxCAwAmfgUlIr0UL7fcy9y6L/zF21KzFLvXalD5foh8st\nXi295fg53O/vq6eWkhrKzNEFvsguac3IaUScGxHbp2y3zH2kJEmS+qVts/VvBk5g4s+JJwvGIkmS\npCnalpw+mZkPlg5CkiRJ02vNaf3aIRFxT0TcFhGbIuLZpQOSJEnShDYlp9cDbwHWAWcCzweujog9\nSwYlSZKkCa05rZ+ZW3oe3hwRNwB3AqcAnyoTlSRJknq1JjmdKjPHIuJHwMGz73kFsHpK2RHAkUsT\nmCRJUl/8gGqueK/HSwQySWuT04h4BlVievHse57EzOuc/gCT1PmwneZ2M9UfPZqdfWl+bKf58fdu\nfuxPc1uufelIdnxvn1rntJjWXHMaER+JiLUR8dyIeAVwGfAE0F34q079a0PTs53mZhvNj+00P7bT\n/NhO82M7zc02WkxtGjk9CLgE2Bd4ELgWeHlmPlw0KkmSJD2lNclpZnZKxyBJkqTZtea0viQ12x3A\nB4CtheOQpLJaM3K6APUU/Ydm2eVxqguHNbsmtlMu8LiYe5cF/cylbKNdiblpmtiXFsvDVO/VP7Lr\ndRxvp7uormIa2sXXG7fQ3xso0w/nine2/rSSfm921Ur6vduVPjybudpoqb47ZrPQn/lU3jN1qaK+\nicyleqOWt4h4E/CZ0nFIkiQVcGpmXlLiB5ucziAi9qW6m9QdNGHRL0kr3SDV+i3/J3DVIr3mu4DX\nAy9bpNeTtPKtBp4HbCk1adzkVJJ2QUS8CrgSeF1mfmHKc28CNgHHZua353id4+vXeQPwIqrbLO8H\nfBP448y8rWff44D/HTgG2B/4GfD/Av8lMx+v9/kU8J+ozgmOn9/LzNxtV+orSUvNa04laRdk5tcj\n4m7gVOALU54+FfjxXIlpjwDeA/wS+AgwQDX6uQk4tme//w14GrCR6mLVo4Gzgd+gSm4B/gp4FnBi\nHYcXUUpaFkxOJWnXbQKGI2KvzPwngIjYD/h3wAd38rX2AI7KzF/Wr/Nz4PyIODwzb6n3eWdm/kvP\nMX8bEbcB/1dEHJSZP83Mb9e3aD4xM3fhZiOS1F8uJSVJu+5iquu0Xt9T9kZgN3Z+YuX/GE9Ma9dQ\njXq+YLygNzGNiKfX18h/i+oz/SU7+fMkqVFMTiVpF2XmPwDfoTp9Pu5NwPWZeftOvtzdUx7/Y/3v\n/zJeEBHPjogLI+Jh4J+p1ov6OtX1pQM7+fMkqVFMThcgIt4WET+JiF9ExPUR4UzYHhFxbkRsn7Ld\nMveRK1tErImIzRFxT90m66fZ57yIuDciHouIv4+Ig0vEWtJc7RQRn5qmf325VLw9LgaOj4hnRcQL\ngZcDn17A6/xyhvIAiIhVwChVIrwnMEY1aepd9T69n+vHAKva2p8i4j0RcUNEPBoRD0TEZRFx6JR9\nmtqf+iYizoyImyJirN6ui4iTpuzjZ9Mc7WRfWjwmpzspIt4AfBQ4l+r02U3Alvr6Mk24mWoW8QH1\ndlzZcBphT+B7wFuZZlXliHgX8HbgDKoJLtuo+tav9jPIBpi1nWqXM7l/NeH2xJcC26lieRPwr8Bn\nl+DnHEk1i/9vqD6D1lIlqP9H7051f/rNOqa29qc1wCeokvQTgd2Br0TE06bs18T+1E93U/1xM0h1\nx4Yrgc0RcTj42dRj1naqtb0vLQonRO28YeCTmXkxVH9JAf8BOA3485KBNcyTmflg6SCaJDOvAK4A\niIjpZk6/A/hgZv5/9T6/DzwA/A5Lk+Q00jzaCeBfmta/MvPhiLgc+D2q60+vyMxHluBH/ZIqaf9W\nZm4FiIi3UJ3a396z3zuAa4FXU90qqnX9KTNP7n1ct9PPqBKLa3uealx/6qfM/NKUovdFxFlUSf0t\n+NkEzKudoOV9abE4croTImJ3qg+1r42XZbVQ7FeZvMyL4JD6tOxtEbEpIp5dOqAmi4jnU/2V3du3\nHgW+jX1rOq+qT9PeGhEbI+LXSgdUu5hqtPIQqhn8S+FW4Dbgo/Vp67cBX+zdoac/fZnqVP8ngN8G\nbqfd/WkfqsR+6h8NTe1PfRcRqyLijVSrRlztZ9P0prZTz1P2pUXgyOnO2Y9q9u0DU8ofoFo0W5Xr\ngbcA/wAcCLyf6kPuiMzcVjCuJjuA6ktzur51QP/DabTLgc8DPwFeCPw34MsRcWyWv6vIF6kmMAWw\neQHHzxT/U+WZ+WRE/DbwceDdVHew+wXwXarTjTDRny4BnkO1csCpVAMS319AXMtePQp/PnBtz5Jc\n0Oz+1DcRcQTVig+rgceAUzLztog4Fj+bnjJTO9VP25cWicmpFl1mbul5eHNE3ADcCZwCfKpMVFop\nMrP3NOIPI+IHVCOJr2Lxbvu5UNuBJ4EvZOa/7syBmfkNqj9+p5bfObW8Xh1gHUBE/GX9//WZeV9d\nduzErvmfgf9cl/8/O1WblWUjcDjwyt7ChvenfroVOIpqtYfXA5dGddcyTTZtO2XmjfalxeNp/Z3z\nENX1XvtPKd8fuL//4SwPmTkG/Aho3ezOnXA/1WibfWsnZeZPqH43m9C/Xkd1huXifvywiLgAOBl4\n1XhiWrM/9ZilnXbQsP7UN5n5ZGbeXidZ76U6bX8W9qVJZmmn6fZtZV9aDCanOyEznwBGgBPGy+pT\nRScA15WKq+ki4hlUv5yzfim0Wf0hdj+T+9beVBfa27dmEREHAftSsH9FxNER8UdUK3mMZua1Pc/t\nHhH7z7GtXsDPvAB4LfDqzLyr9zn704TZ2mmG/Yv3p4ZYBexmX5rTKqY54wH2pV3haf2d9zHgwogY\nAW6gmr3/dODCkkE1SUR8hOrauzup7vX9AeAJoNW3UIyIPamS9PEZ6C+IiKOARzLzbqrr4d4XET8G\n7qC67eVP2fF+7SvabO1Ub+dSXdd1f73fn1GNzG/Z8dX65iyqazpvBP5gynOvYPZTelkfM+/R1ojY\nSLVEzXpgW0SMj2qNZebj9f9b35/maqe6rzWxP/VVRHyY6nrJu4C9qPryWuBD9S6t70swezvZlxZX\neI3uzouItwLvpDqt8T3g7Mz8btmomiMiulTrC+5LtbzNtcB767/AW6u+fusqdpz0clFmnlbv836q\ntQT3obpt5dsy88f9jLO02dqJau3T/wm8mKqN7qX64D+nqcu3RMQA1Sofs/lhZk6dcDLba25n+slT\nfzC+zF293/tpcX+aq53qEetl1Z+WQkT8LfAaqgmsY1ST5v40M6/s2ef9tLgvweztZF9aXCanM4jq\nXtXrqP5KfHz2vSVJklaE1cDzgC2Z+XCJADytP7N1wGdKByFJklTAqVTL0fWdyenM7gDYtGkThx12\n2LQ7DA8Ps2HDhn7GtCzZTnOzjebHdpof22l+bKf5sZ3mtpLaaOvWrbz5zW+GOg8qoXXJaX03lT+h\nWjz4JqrrRb8zza6PAxx22GEMDg5O8zQMDAzM+Jwm2E5zs43mx3aaH9tpfmyn+bGd5rZC26jYJY2t\nWkoqIt5AtdTLucBLqJLTLRGxX9HAJEmSBLQsOaVa9umTmXlxZt4KnEl1+7HTyoYlSZIkaFFyGhG7\nUy3n8rXxsvpet18Fjp3pOEmSJPVPa5JTqlsK7gZMXUvwAarrT3dap9PZ1ZhawXaam200P7bT/NhO\n82M7zY/tNDfbaHG1Zp3TiDgQuAc4NjO/3VP+Z8DazDx2yv6DwMjatWsZGBiY9FqdTseOKEmSlrVu\nt0u3O/nmjWNjY1x99dUAQ5k5WiKuNiWnu1NdX/q7mbm5p/xCYCAzXzdl/0FgZGRkZCXOwJMkSdrB\n6OgoQ0NDUDA5bc1p/cx8AhgBThgvi4ioH19XKi5JkiRNaNs6px8DLoyIEeAGqtn7TwcuLBmUJEmS\nKq1KTjPzs/WapucB+wPfA9Zl5oNlI5MkSRK0LDkFyMyNwMbScUiSJGlHrbnmVJIkSc1ncipJkqTG\nMDmVJElSY7QqOY2INRGxOSLuiYjtEbG+dEySJEma0KrkFNiTaob+W4F23H1AkiRpGWnVbP3MvAK4\nAp5agF+SJEkN0raRU0mSJDWYyakkSZIaw+RUkiRJjdGqa04XYnh4mIGBgUllnU6HTqdTKCJJkqRd\n1+126Xa7k8rGxsYKRTMhMts5aT0itgO/k5mbZ3h+EBgZGRlhcHCwv8FJkiQVMDo6ytDQEMBQZo6W\niKFVI6cRsSdwMDA+U/8FEXEU8Ehm3l0uMkmSJEHLklPgpcBVVGucJvDRuvwi4LRSQUmSJKnSquQ0\nM7+Bk8AkSZIay0RNkiRJjWFyKkmSpMYwOZUkSVJjmJxKkiSpMVqTnEbEeyLihoh4NCIeiIjLIuLQ\n0nFJkiRpQmuSU2AN8AngGOBEYHfgKxHxtKJRSZIk6SmtWUoqM0/ufRwRbwF+BgwB15aISZIkSZO1\naeR0qn2oFuJ/pHQgkiRJqrQyOY2IAM4Hrs3MW0rHI0mSpEprTutPsRE4HHhl6UAkSZI0oXXJaURc\nAJwMrMnM++baf3h4mIGBgUllnU6HTqezRBFKkiQtvW63S7fbnVQ2NjZWKJoJkZmlY+ibOjF9LXB8\nZt4+x76DwMjIyAiDg4N9iU+SJKmk0dFRhoaGAIYyc7REDK0ZOY2IjUAHWA9si4j966fGMvPxcpFJ\nkiRpXJsmRJ0J7A18Hbi3ZzulYEySJEnq0ZqR08xsUyIuSZK0LJmwSZIkqTFMTiVJktQYJqeSJElq\nDJNTSZIkNUZrktOIODMiboqIsXq7LiJOKh2XJEmSJrQmOQXuBt4FDAJDwJXA5og4vGhUkiRJekqb\nlpL60pSi90XEWcAxwC0FQpIkSdIUrUlOe0XEKqrF9/cArikcjiRJkmqtSk4j4gjgW8Bq4DHglMz8\ncdmoJEmSNK5N15wC3AocBRwNXABcGhEvKRuSJEmSxrVq5DQznwRurx/eGBFHA2cBZ8x0zPDwMAMD\nA5PKOp0OnU5nyeKUJElaat1ul263O6lsbGysUDQTIjNLx1BMRHwNuCMz/3Ca5waBkZGREQYHB/sf\nnCRJUp+Njo4yNDQEMJSZoyViaM3IaUR8GLgcuAvYCzgVWAt8qGRckiRJmtCa5BT4deAi4EBgDPg+\nsC4zryoalSRJkp7SmuQ0M08vHYMkSZJm17bZ+pIkSWowk1NJkiQ1hsmpJEmSGsPkVJIkSY3R2uQ0\nIt4dEdsj4mOlY5EkSVKllclpRLyM6q5QN5WORZIkSRNal5xGxDOATcDpwM8LhyNJkqQerUtOgb8A\nvpiZV5YORJIkSZO1ZhF+gIh4I/Bi4KWlY5EkSdKOWpOcRsRBwPnAiZn5xHyPGx4eZmBgYFJZp9Oh\n0+kscoSSJEn90+126Xa7k8rGxsYKRTMhMrN0DH0REa8F/g74JRB18W5A1mV7ZE9jRMQgMDIyMsLg\n4GC/w5UkSeq70dFRhoaGAIYyc7REDK0ZOQW+Chw5pexCYCvwp9mWLF2SJKnBWpOcZuY24JbesojY\nBjycmVvLRCVJkqRebZyt38vRUkmSpAZpzcjpdDLzNaVjkCRJ0oS2j5xKkiSpQUxOJUmS1Bgmp5Ik\nSWqM1iSnEXFuRGyfst0y95GSJEnql7ZNiLoZOIGJRfifLBiLJEmSpmhbcvpkZj5YOghJkiRNrzWn\n9WuHRMQ9EXFbRGyKiGeXDkiSJEkT2pScXg+8BVgHnAk8H7g6IvYsGZQkSZImtOa0fmZu6Xl4c0Tc\nANwJnAJ8qkxUkiRJ6tWa5HSqzByLiB8BB8+23/DwMAMDA5PKOp0OnU5nKcOTJElaUt1ul263O6ls\nbGysUDQTIrOdt5ePiGcAdwHnZOYF0zw/CIyMjIwwODjY9/gkSZL6bXR0lKGhIYChzBwtEUNrrjmN\niI9ExNqIeG5EvAK4DHgC6M5x6Iym/rWh6dlOc7ON5sd2mh/baX5sp/mxneZmGy2u1iSnwEHAJcCt\nwKXAg8DLM/Phhb6gnXF+bKe52UbzYzvNj+00P7bT/NhOc7ONFldrrjnNTC8SlSRJarg2jZxKkiSp\n4UxOJanhPve5z7Fq1Sq+8IUv7PDcUUcdxapVq/jGN76xw3PPec5zOO644/oRoiQtmtac1l+A1QBb\nt26dcYexsTFGR4tMZFtWbKe52Ubz09Z22nvvvQH4/Oc/z7OfPXFju23btvHDH/6QX/mVX+Fzn/sc\ne+21F1C10+WXX85Pf/pTTjjhhFa22Xy0tT/tLNtpbiupjXryntWlYmjtUlJziYg3AZ8pHYckSVIB\np2bmJSV+sMnpDCJiX6pbnd4BPF42GknLyMFUK4IMA9fUZf8G2ES1Wsibe/b9BLAX1a2V5/IB4LeA\ntVTL4AH8MfBq4CLgnfX/x70L+F3gBOCfdr4aklpqNfA8YMuurGi0KzytP4P6DSnyF4Ok5SsibgT+\nCviN8QWsI2INsB04BPhRZv5zRATwb4G/ms9C1xGxGTgZ2CMzv12XvRD4GtV6zR8E/jUzb66fOwS4\nNTN3vBhVkmZ3Xckf7oQoSVpEWZ2O+iawpqd4DdWNPxJ4RV32YmBv4Np5vvS1QADHAUTEbsAxwLWZ\neTvwQM9zzwCO3InXlqTGMDmVpMV3DTAYEU+rHx8HfBm4iYmkdXw0dV4JZGZuBR6uXwuq5PbpTIxw\nXAe8sv7/K4Dd5vvaktQkJqeStPiuAXYHjo2IQ4Fn1mVXM5GcHgfckpk/34nXvQ54ef3/VwI/y8yf\n9Dz3yp7nEpNTScuQyekCRMTbIuInEfGLiLg+Il5WOqYmiYhzI2L7lO2W0nGVFhFrImJzRNxTt8n6\nafY5LyLujYjHIuLvI+LgErGWNFc7RcSnpulfXy4V7wy+SzWRci1VMvqzzPwxVYJ6dET8al1+zcwv\nMa1rgYGIOB84F9gvIh6IiMuAu4HnRsSBVMnpvcBpbe5PEfGeiLghIh4db6f6j4XefZZDf1pSEXFm\nRNwUEWP1dl1EnDRlHz+b5mgn+9LiMTndSRHxBuCjVF8ML6E6TbclIvYrGljz3AzsDxxQb64EDnsC\n3wPeSjWqNUlEvAt4O3AGcDSwjapv/Wo/g2yAWdupdjmT+1ejbk+cmU8ANzCRnI4nodcAewCnUsV/\n9U6+9PhI6In1vx+t/7878OfAv1DN2D+G6hKAtvenNVQrIhzDRDt9pedyi3GN7k99cDfV6g6DwBBw\nJbA5Ig4HP5t6zNpOtbb3pcWRmW47sQHXA/+953EAPwXeWTq2pmxUifto6TiavFFda7h+Stm9wHDP\n472BXwCnlI63Ye30KeDvSsc2j9g/SPUlfgdwdk/5D6mWlPol8KydfM3dgceoJlz9Enh5Xb5f3Vbf\n73nu5/anHdpvvJ2OW279qUBbPQz8Qf1/P5vm1072pUXaHDndCRGxO9VfS18bL8uqR34VOLZUXA11\nSH1a9raI2BQRz577kPaKiOdT/ZXd27ceBb6NfWs6r6pP094aERsj4tdKBzSNa4CnAc9m8un7q4FD\ngTsy896decGsRmS/Q9Un/gUYqZ/ah2qU+btM9Je9sT9NNd5Oj0wpXw79qS8iYlVEvJFqhP9qP5um\nN7Wdep6yLy0Ck9Odsx/VDNgHppQ/QPXLq8r1VIuKrwPOBJ5P9SG3Z8mgGu4Aqi9N+9bcLgd+H3gN\n1cLzxwNfrtcNbZLrqEYwH6W6/GfcNVTv9c6e0h93bX38dzPzibre59flm+vntmF/mqS3nTKz9xr4\n5dKfllSJQbepAAAgAElEQVREHBER/0T1R88nqUZFb8PPpklmaSewLy0aF+HXosvMLT0Pb46IG4A7\ngVOoTntIC5aZn+15+MOI+AFwG/Aq4KoiQU0jM/+Z6jT81PJL2IUbfGTme4H39hRtBA4HXpmZ9wG7\nRcSxOFN/qqfaqbdwufSnPrgVOAoYAF4PXBoRx5cNqZGmbafMvNG+tHgcOd05D1GNhOw/pXx/4P7+\nh7M8ZOYY8COq2zpqevdTXb9s39pJWS2l9BAt7F8RcQHVXaNeVSem4+xPPWZppx20tT9l5pOZeXud\nZL2X6rT9WdiXJpmlnabbt5V9aTGYnO6E+nqvEap7VQNPnSo6gcK3+mqyqO5WczAw65dCm9UfYvcz\nuW/tTTXL2L41i4g4CNiXZdq/ImLPiNh/jm2Hz+o64Xot8OrMvKv3OfvThNnaaYb9l3V/WkSrgN3s\nS3NaRXW53w7sSwvnaf2d9zHgwogYoVoqZpjqLi0XlgyqSSLiI8AXqU7l/wbwAeAJqvt/t1Z9ze3B\nVKMQAC+IiKOARzLzbqrr4d4XET+mmuH9QaqVIL5QINxiZmunejsX+DzVF+bBwJ9Rjcxv2fHVloU/\noarTTJLquu2nEquI2Ei1RM16YFtEjI9qjWXm4/X/W9+f5mqnuq+ttP600yLiw1TXS94F7EW11Nla\n4EP1Lq3vSzB7O9mXFpfJ6U7KzM/Wa5qeR3Va43vAusx8sGxkjXIQ1TV1+wIPUl379vLMfLhoVOW9\nlOq6o6y3j9blFwGnZeafR8TTqS6y34dq4sy/z8x/LRFsQbO101uB36SadLAP1RI3W4Bz6jMby9FF\nzL0Y/9TTp2dStc3Xp5T/AXAxgP0JmLudfsnK608L8etU/fBAYIxqSbJ1mXkV2Jd6zNhOEbEa+9Ki\niWolJC1XEbEv1az4O6juSCNJkpppNfA8YIsDNjNz5HT5Wwd8pnQQkiRp3k5lF1btWOlMThsoIt5G\ndR3aAVTrI56dmd+ZYfc7ADZt2sRhhx026Ynh4WE2bNiwhJE2l3VvZ92h3fW37ta9jZZT/bdu3cqb\n3/xmqL+7NT2T04aJiDdQXWN3BhMTrrZExKGZ+dA0hzwOcNhhhzE4ODjpiYGBgR3K2sK6t7Pu0O76\nW3fr3kbLtP5ehjcLl5JqnmHgk5l5cWbeSnVB/2PAaWXDkiRJWnompw0SEbsDQ0y+h3ECX6XF9zCW\nJEntYXLaLPtRLebrPYwlSVIrmZyuYJ1Op3QIxVj39mpz/a17O7W57mD9VyLXOW2Q+rT+Y8DvZubm\nnvILgYHMfN00xwwCI2vXrmVgYGDSc51Ox19aSZIK6Ha7dLuTb4w4NjbG1VdfDTCUmaNFAlsGTE4b\nJiKuB76dme+oHwfVrdI+npkfmWb/QWBkZGRkOc5WlCSpNUZHRxkaGgKT01m5lFTzfAy4MCJGmFhK\n6unAhSWDkiRJ6geT04bJzM9GxH7AecD+wPeo7t37YNnIJEmSlp7JaQNl5kZgY+k4JEmS+s3Z+pIk\nSWoMk9OGiYg1EbE5Iu6JiO0Rsb50TJIkSf1icto8e1JdZ/pWwKUUJElSq3jNacNk5hXAFfDUMlKS\nJEmt4cipJEmSGsPkVJIkSY1hcipJkqTG8JrTFWJ4eJiBgYFJZZ1Oh06nUygiSZLaq9vt0u12J5WN\njY0VimZ5iUwnhDdVRGwHficzN8+yzyAwMjIywuDgYP+CkyRJO2V0dJShoSGAocwcLR1PUzly2jAR\nsSdwMDA+U/8FEXEU8Ehm3l0uMkmSpKVncto8LwWuolrjNIGP1uUXAaeVCkqSJKkfTE4bJjO/gRPV\nJElSS5kESZIkqTFMTiVJktQYJqcNEhHviYgbIuLRiHggIi6LiENLxyVJktQvJqfNsgb4BHAMcCKw\nO/CViHha0agkSZL6xAlRDZKZJ/c+joi3AD8DhoBrS8QkSZLUT46cNts+VMtJPVI6EEmSpH4wOW2o\niAjgfODazLyldDySJEn94Gn95toIHA68snQgkiRJ/WJy2kARcQFwMrAmM++bzzHDw8MMDAxMKut0\nOnQ6nSWIUJIkzabb7dLtdieVjY2NFYpmeYnMLB2DetSJ6WuB4zPz9nnsPwiMjIyMMDg4uOTxSZKk\nhRkdHWVoaAhgKDNHS8fTVI6cNkhEbAQ6wHpgW0TsXz81lpmPl4tMkiSpP5wQ1SxnAnsDXwfu7dlO\nKRiTJElS3zhy2iCZ6R8LkiSp1UyGJEmS1Bgmp5IkSWoMk9MGiYgzI+KmiBirt+si4qTScUmSJPWL\nyWmz3A28CxgEhoArgc0RcXjRqCRJkvrECVENkplfmlL0vog4CzgG8BamkiRpxTM5baiIWEW1hNQe\nwDWFw5EkSeoLk9OGiYgjgG8Bq4HHgFMy88dlo5IkSeoPrzltnluBo4CjgQuASyPiJWVDkiRJ6g9H\nThsmM58Ebq8f3hgRRwNnAWfMdtzw8DADAwOTyjqdDp1OZ0nilCRJM+t2u3S73UllY2NjhaJZXiIz\nS8egWUTE14A7MvMPZ3h+EBgZGRlhcHCwv8FJkqR5Gx0dZWhoCGAoM0dLx9NUjpw2SER8GLgcuAvY\nCzgVWAt8qGRckiRJ/WJy2iy/DlwEHAiMAd8H1mXmVUWjkiRJ6hOT0wbJzNNLxyBJklSSs/UlSZLU\nGCanDRYR746I7RHxsdKxSJIk9YPJaUNFxMuolo+6qXQskiRJ/WJy2kAR8QxgE3A68PPC4UiSJPWN\nyWkz/QXwxcy8snQgkiRJ/eRs/YaJiDcCLwZeWjoWSZKkfjM5bZCIOAg4HzgxM58oHY8kSVK/mZw2\nyxDwTGA0IqIu2w1YGxFvB/bIGe43Ozw8zMDAwKSyTqdDp9NZynglSdI0ut0u3W53UtnY2FihaJaX\nmCHXUQERsSfw3CnFFwJbgT/NzK3THDMIjIyMjDA4OLj0QUqSpAUZHR1laGgIYCgzR0vH01SOnDZI\nZm4Dbukti4htwMPTJaaSJEkrjbP1m8+hbUmS1BqOnDZcZr6mdAySJEn94sipJEmSGsPkVJIkSY1h\nctogEXFuRGyfst0y95GSJEkrg9ecNs/NwAnA+DqnTxaMRZIkqa9MTpvnycx8sHQQkiRJJXhav3kO\niYh7IuK2iNgUEc8uHZAkSVK/mJw2y/XAW4B1wJnA84Gr6ztHSZIkrXie1m+QzNzS8/DmiLgBuBM4\nBfhUmagkSZL6x+S0wTJzLCJ+BBw8177Dw8MMDAxMKut0OnQ6naUKT5IkzaDb7dLtdieVjY2NFYpm\neYlM747ZVBHxDOAu4JzMvGCGfQaBkZGREQYHB/sanyRJmr/R0VGGhoYAhjJztHQ8TeU1pw0SER+J\niLUR8dyIeAVwGfAE0J3jUEmSpBXB0/rNchBwCbAv8CBwLfDyzHy4aFSSJEl9YnLaIJnpBaKSJKnV\nPK0vSZKkxjA5lSRJUmOYnDZMRDwrIj4dEQ9FxGMRcVM9I1+SJGnF85rTBomIfYBvAl+jukvUQ8Ah\nwD+WjEuSJKlfTE6b5d3AXZl5ek/ZnaWCkSRJ6jdP6zfLfwS+GxGfjYgHImI0Ik6f8yhJkqQVwuS0\nWV4AnAX8A/BbwF8CH4+I3ysalSRJUp94Wr9ZVgE3ZOZ/rR/fFBFHAGcCny4XliRJUn84ctos9wFb\np5RtBZ4z14HDw8OsX79+0nb22WcvSZDLQbfb3ju+trnu0O76W/d2anPdobn173a7O3wvDw8Plw5r\nWTA5bZZvAi+aUvYi5jEpasOGDWzevHnSdued7Z1L1dQPq35oc92h3fW37u3U5rpDc+vf6XR2+F7e\nsGFD6bCWBZPTZtkAvDwi3hMRL4yINwGnAxcUjkuSJKkvTE4bJDO/C7wO6AA/AN4LvCMzLy0amCRJ\nUp84IaphMvPLwJdLxyFJklSCyenytxpg69ap86hgbGyM0dHRvgfUBNa9nXWHdtffulv3NlpO9e/5\nrl5dMo6mi8wsHYN2QX1d6mdKxyFJkubt1My8pHQQTWVyusxFxL7AOuAO4PGy0UiawcHApcAwcE1d\n9m+ATcCtwJt79v0EsBfwlnm87h8DfwS8vv73FcA9U15PUnOsBp4HbMnMhwvH0lie1l/m6s7tX19S\ng0XEjcBfAb+RmaN12RpgO3AI8KPM/OeICODfAn81vt8cr3tv/d/zgB8B76QadFge5zildrqudABN\nZ3IqSUssMzMivgms6SleA1wGvJZqxPMrwIuBvYFrd/JH3JiZ3uZY0orgUlKS1B/XAIMR8bT68XFU\nK3PcxETSOj6aujPJaQKfXKwgJak0R04lqT+uAXYHjo2InwLPrMuOYCI5PQ64JTN/vpOv/ZNFi1KS\nCnPkVJL647tUkxbXUiWjP8vMH1MlqEdHxK/W5dfM/BIz+sWiRSlJhZmcrkAR8baI+ElE/CIiro+I\nl5WOqR8i4tyI2D5lu6V0XEshItZExOaIuKeu5/pp9jkvIu6NiMci4u8j4uASsS62ueoeEZ+aph8U\nv7FFZj4B3MBEcjqehF4D7AGcCuwPXD3Ta9S3Nr4hIh6NiAeAN86w34p776fWPSIui4hDp+zTyPd+\nMUTEmRFxU0SM1dt1EXHSlH1W3PsOc9d9Jb/vbWVyusJExBuAjwLnAi+hup5tS0TsVzSw/rmZ6gv+\ngHo7rmw4S2ZP4HvAW6muOZwkIt4FvB04Azga2EbVD361n0EukVnrXrucyf2g05/Q5nQNcAzwqvr/\n4ytu3Aq8i6o+s42crqFaauoY4ESqz/BV9CzovYLf+6l13x34Ss81vOOa+t7vqrup+sggMARcCWyO\niMNhRb/vMEfdayv1fW+nzHRbQRtwPfDfex4H8FPgnaVj60PdzwVGS8dRoN7bgfVTyu4Fhnse7011\n6veU0vH2oe6fAv6udGwzxPtbdcy/BF7cU/6XdfltO/l6f0aV0P77Fr73+9VtdtxyeO+XqA0eBv6g\nTe/7DHVv1fvehs2R0xUkInan+qvya+NlWf3mfhU4tlRcfXZIfbr3tojYFBHPLh1Qv0XE86lGDnr7\nwaPAt2lPP3hVfer31ojYGBG/Vjqg2nVUiemjVGc1xl1DlWTOeEp/BuMjpj+H1r33+1C12SNTypv6\n3i+aiFgVEW+kuhzk6ja971Pr3vPUin/f28TZ+ivLfsBuwANTyh8AXtT/cPrueqq76vwDcCDwfqoP\n7iMyc1vBuPrtAKov7en6wQH9D6fvLgc+TzWD/YXAfwO+HBHH1n+sFZOZ/0x1Onpq+SXs5M006gX7\nXwhcnZnfqotb8d7XdT8fuDYze68rb+x7vxgi4gjgW1R/lDxGNSp6W0Qcywp/32eqe/30in7f28jk\nVCtGZm7peXhzRNwA3AmcQnXaRy2QmZ/tefjDiPgBcBvVdZ5XFQlqaWwEDgdeWTqQAqatewve+1uB\no4ABqlvWXhoRx5cNqW+mrXtm3tiC9711PK2/sjxEdcpw/ynl+wP39z+csjJzjOqWjitixupOuJ/q\nWmP7AZCZP6H63Vg2/SAi9oyI/WfZ/m/gZOBVmXlfz6Er/r2PiAuYvu47WI7v/Wwy88nMvL1OyN5L\nddr+LFrwvs9S9+n2XVHvexuZnK4gWS1VMwKcMF5Wn/46gRbeyzcinkH14TTrF9hKU38w38/kfrA3\n1SznNvaDg4B9WV794E+o4p1uux84DTg1M+/qPWilv/d1Yvpa4NVT6z7D/svxvd8Zq4DdVvr7PoNV\nVJex7aAF7/uK52n9ledjwIURMUK1puIw8HTgwpJB9UNEfAT4ItWp/N8APgA8AXRLxrUUImJPqsQ7\n6qIXRMRRwCOZeTfV9Xjvi4gfA3cAH6RateELBcJdVLPVvd7Opbr+7P56vz+jGkHfsuOrNdZFTL+k\n1DuA1wDvA+6IiPGRsrHMfLz+/4p87yNiI9XyQOuBbVPrXveLlfDeTysiPkx1beVdwF5U6+KuBT5U\n77Ii33eYve4r/X1vrdLLBbgt/ka1/uMdVMuIfAt4aemY+lTvLtWH8S+oPsQuAZ5fOq4lquvxTCxJ\n1Lv9j5593k+1vMxjVB/SB5eOe6nrTjVZ4gqqL6nHgduplml6Zum4F6nu09X7l8DvT9lvxb33c9W9\nBe/939Z1+kVdx68Ar1np7/tcdV/p73tbt6jfXC1TEbEvsI4qGX189r0lSVJBq4HnAVuyugGHpuFp\n/QaKiLdRXXN2ANVaiGdn5ndm2H0d8Jl+xSZJknbZqezk8nFtYnLaMD23Hz2DiWtGt0TEoZn50DSH\n3AGwadMmDjvssElPDA8Ps2HDhqUNuKGsezvrDu2uv3W37m20nOq/detW3vzmN0P93a3pmZw2zzDw\nycy8GCAizgT+A9Xs3D+fZv/HAQ477DAGBwcnPTEwMLBDWVtY93bWHdpdf+tu3dtomdbfy/Bm4VJS\nDeLtRyVJUtuZnDbLbLcfXRG3oJMkSZqNyakkSZIaw2tOm2XBtx8dHh5mYGBgUtlzn/vcRQ1uOel0\nOqVDKKbNdYd219+6t1Ob6w7NrX+326XbnXwPmLGxsULRLC+uc9owEXE98O3MfEf9OKgWlP94Zn5k\nmv0HgZGRkZHleEG4JEmtMTo6ytDQEMBQZo6WjqepHDltntbeflSSJMnktGEy87MRsR9wHtXp/O8B\n6zLzwbKRSZIkLT2T0wbKzI3AxtJxSJIk9Zuz9SVJktQYJqeSJElqDJPThomINRGxOSLuiYjtEbG+\ndEySJEn9YnLaPHtSTYJ6K+A6X5IkqVWcENUwmXkFcAU8tcapJElSazhyKkmSpMYwOZUkSVJjmJxK\nkiSpMbzmdIUYHh5mYGBgUlmn06HT6RSKSJKk9up2u3S73UllY2NjhaJZXiLTCeFNFRHbgd/JzM2z\n7DMIjIyMjDA4ONi/4CRJ0k4ZHR1laGgIYCgzR0vH01SOnDZMROwJHAyMz9R/QUQcBTySmXeXi0yS\nJGnpmZw2z0uBq6jWOE3go3X5RcBppYKSJEnqB5PThsnMb+BENUmS1FImQZIkSWoMk9MGiYj3RMQN\nEfFoRDwQEZdFxKGl45IkSeoXk9NmWQN8AjgGOBHYHfhKRDytaFSSJEl94jWnDZKZJ/c+joi3AD8D\nhoBrS8QkSZLUT46cNts+VDP2HykdiCRJUj+YnDZURARwPnBtZt5SOh5JkqR+8LR+c20EDgdeWToQ\nSZKkfjE5baCIuAA4GViTmffN55jh4WEGBgYmlXU6HTqdzhJEKEmSZtPtdul2u5PKxsbGCkWzvERm\nlo5BPerE9LXA8Zl5+zz2HwRGRkZGGBwcXPL4JEnSwoyOjjI0NAQwlJmjpeNpKkdOGyQiNgIdYD2w\nLSL2r58ay8zHy0UmSZLUH06IapYzgb2BrwP39mynFIxJkiSpbxw5bZDM9I8FSZLUaiZDkiRJagyT\nU0mSJDWGyWmDRMSZEXFTRIzV23URcVLpuCRJkvrF5LRZ7gbeBQwCQ8CVwOaIOLxoVJIkSX3ihKgG\nycwvTSl6X0ScBRwDeAtTSZK04pmcNlRErKJaQmoP4JrC4UiSJPWFyWnDRMQRwLeA1cBjwCmZ+eOy\nUUmSJPWH15w2z63AUcDRwAXApRHxkrIhSZIk9Ycjpw2TmU8Ct9cPb4yIo4GzgDNmO254eJiBgYFJ\nZZ1Oh06nsyRxSpKkmXW7Xbrd7qSysbGxQtEsL5GZpWPQLCLia8AdmfmHMzw/CIyMjIwwODjY3+Ak\nSdK8jY6OMjQ0BDCUmaOl42kqR04bJCI+DFwO3AXsBZwKrAU+VDIuSZKkfjE5bZZfBy4CDgTGgO8D\n6zLzqqJRSZIk9YnJaYNk5umlY5AkSSrJ2fqSJElqDJNTSZIkNYbJaYNFxLsjYntEfKx0LJIkSf1g\nctpQEfEyqrVNbyodiyRJUr+YnDZQRDwD2AScDvy8cDiSJEl9Y3LaTH8BfDEzrywdiCRJUj+5lFTD\nRMQbgRcDLy0diyRJUr+ZnDZIRBwEnA+cmJlPlI5HkiSp30xOm2UIeCYwGhFRl+0GrI2ItwN7ZGZO\nd+Dw8DADAwOTyjqdDp1OZynjlSRJ0+h2u3S73UllY2NjhaJZXmKGXEcFRMSewHOnFF8IbAX+NDO3\nTnPMIDAyMjLC4ODg0gcpSZIWZHR0lKGhIYChzBwtHU9TOXLaIJm5DbiltywitgEPT5eYSpIkrTTO\n1m8+h7YlSVJrOHLacJn5mtIxSJIk9Ysjp5IkSWoMk9MGiYhzI2L7lO2WuY+UJElaGTyt3zw3AycA\n40tJPVkwFkmSpL4yOW2eJzPzwdJBSJIkleBp/eY5JCLuiYjbImJTRDy7dECSJEn9YnLaLNcDbwHW\nAWcCzweurhfnlyRJWvE8rd8gmbml5+HNEXEDcCdwCvCpMlFJkiT1j8lpg2XmWET8CDh4rn2Hh4cZ\nGBiYVNbpdOh0OksVniRJmkG326Xb7U4qGxsbKxTN8hKZ3oCoqSLiGcBdwDmZecEM+wwCIyMjIwwO\nDvY1PkmSNH+jo6MMDQ0BDGXmaOl4msprThskIj4SEWsj4rkR8QrgMuAJoDvHoZIkSSuCp/Wb5SDg\nEmBf4EHgWuDlmflw0agkSZL6xOS0QTLTC0QlSVKreVpfkiRJjWFyKkmSpMYwOW2YiHhWRHw6Ih6K\niMci4qZ6Rr4kSdKK5zWnDRIR+wDfBL5GdZeoh4BDgH8sGZckSVK/mJw2y7uBuzLz9J6yO0sFI0mS\n1G+e1m+W/wh8NyI+GxEPRMRoRJw+51GSJEkrhMlps7wAOAv4B+C3gL8EPh4Rv1c0KkmSpD7xtH6z\nrAJuyMz/Wj++KSKOAM4EPl0uLEmSpP5w5LRZ7gO2TinbCjxnrgOHh4dZv379pO3ss89ekiCXg263\nvXd8bXPdod31t+7t1Oa6Q3Pr3+12d/heHh4eLh3WsmBy2izfBF40pexFzGNS1IYNG9i8efOk7c47\n2zuXqqkfVv3Q5rpDu+tv3dupzXWH5ta/0+ns8L28YcOG0mEtCyanzbIBeHlEvCciXhgRbwJOBy4o\nHJckSVJfmJw2SGZ+F3gd0AF+ALwXeEdmXlo0MEmSpD5xQlTDZOaXgS+XjkOSJKkEk9PlbzXA1q1T\n51HB2NgYo6OjfQ+oCax7O+sO7a6/dbfubbSc6t/zXb26ZBxNF5lZOgbtgvq61M+UjkOSJM3bqZl5\nSekgmsrkdJmLiH2BdcAdwONlo5EkSbNYDTwP2JKZDxeOpbFMTiVJktQYztaXJElSY5icSpIkqTFM\nTiVJktQYJqeSJElqDJPTFSgi3hYRP4mIX0TE9RHxstIx9UNEnBsR26dst5SO6/9v797j5arqu49/\nvgmXGAgHASUCilyiQqk8JCpN1UCANlGfgheKRG5SkYp4S8tLHpEawAsqhVj0CbYPNZECsTyiFSzk\nIDcDcrMJUhGikoshYLgEOMSQKCS//rHWMXMmczlnJnNmn5nv+/Xar5OzZ+2ZtX97MfzOXmuv1QqS\n3i7pOkmP5fM8ukKZCyQ9LukFST+StH876rq11Tt3SXMrtIOOWNgiL218n6TnJT0h6fuSXlehXMdd\n+8Gce4df+49IekBSX97ukjS9rEzHXXeof+6dfN27lZPTDiPp/cDFwCzgEOABoFfSbm2t2PB5ENgd\nGJ+3t7W3Oi2zA/Az4KPAFlNuSDob+BhwOvAWYB2pHWw3nJVskZrnnt3IwHYwY3iq1nJvB74OHAoc\nBWwL3CTpZf0FOvja1z33rFOv/aPA2cBEYBJwK3CdpAOho6871Dn3rFOve1fyVFIdRtI9wL0R8cn8\nu0j/YV8aEV9ta+VaTNIs4JiImNjuugwnSZuAd0fEdSX7HgcuiojZ+fedgCeAUyLimvbUdOurcu5z\ngZ6IeG/7ajY88h+dTwJTIuLOvK9brn2lc++aaw8gaQ1wVkTM7Zbr3q/s3LvquncD3zntIJK2Jf1V\neUv/vkh/fdwMTG5XvYbZhNzdu1TSlZJe3e4KDTdJ+5DuHJS2g+eBe+mednB47vpdImmOpF3aXaEW\n2Zl09/gZ6LprP+DcS3T8tZc0StLxwPbAwm667uXnXvJSx1/3brJNuytgW9VuwGjSX8ulngBeP/zV\nGXb3AB8Efgm8CjiP9MV9UESsa2O9htt40v+0K7WD8cNfnWF3I3AtsBzYD7gQuEHS5OigrqLcK/I1\n4M6I6B9b3RXXvsq5Q4dfe0kHAXeTVhl6ATguIpZKmkyHX/dq555f7ujr3o2cnFrHiIjekl8flHQf\n8BvgOGBue2plw62sC/MXkn4OLAUOB25rS6VaYw5wIPDWdlekDSqeexdc+yXAwUAPcCzwHUmHtbdK\nw6biuUfE/V1w3buOu/U7y9PARtKg8FK7A6uHvzrtFRF9wK+AjnhidQhWA8LtAICIWE76b6Nj2oGk\nbwDvBA6PiN+WvNTx177GuW+h0659RLwUEctyQvZZUrf9GXTBda9x7pXKdtR170ZOTjtIRLwILAKO\n7N+Xu7+OBO5qV73aRdKOpC+nmv8D6zT5i3k1A9vBTqSnnLuxHewF7EqHtIOcnB0DTI2IlaWvdfq1\nr3XuVcp31LWvYBQwutOvexWjSMPYttAF173juVu/81wCzJO0CLgPmAmMBea1s1LDQdJFwPWkrvw9\ngfOBF4H57axXK0jagZR4K+/aV9LBwDMR8ShpPN65kh4BVgCfB1YBP2hDdbeqWueet1mk8Werc7mv\nkO6g9275biOLpDmkKXKOBtZJ6r9T1hcRG/K/O/La1zv33C46+dp/iTS2ciUwDjgBmAJ8IRfpyOsO\ntc+9069714oIbx22keZ/XAGsJw0gf1O76zRM5z2f9GW8nvQldjWwT7vr1aJzPQzYRBrGUbp9q6TM\necDjpIcHeoH9213vVp876WGJBaT/SW0AlgGXAa9od7230rlXOu+NwMll5Tru2tc79y649pfnc1qf\nz/Em4IhOv+71zr3Tr3u3bp7n1MzMzMwKw2NOzczMzKwwnJyamZmZWWE4OTUzMzOzwnByamZmZmaF\n4SWAOJgAACAASURBVOTUzMzMzArDyamZmZmZFYaTUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZ\nmZlZYTg5NTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJqZmZmZkVhpNTMzMzMysMJ6dmZoCkN0n6iaTf\nSdoo6Y3trpOZWTfapt0VMDNrN0nbAN8FXgA+lX/+pq2VahFJBwDHAXMjYmW762NmVs7JqZkZ7Ae8\nBvhQRMxtd2Va7EBgFnAb4OTUzArH3fpmZrB7/tlXr6CksS2uS6sJiHZXwsysGienZtbVJM0Fbicl\nbN+VtEnSrfm1eZLWStpX0g2SngeuLDn2UEkLJD0naZ2k2yX9eYXP2EPStyStlrRB0oOSTh1k/cZI\nulTSU5Kel/Qf+f02SfpcSbnXSJojaYmkFyQ9LekaSXuXlDkFuCb/ent+j42SppSUeYekhXns7fOS\nfijpwCEF1cysCe7WN7Nu901gFfBZ4J+AnwJP5NeC9D3ZC9wB/D1pPCqSjgBuAP4LOA/YBJwK3Crp\nbRHxX7ncK4F7gY3ApcDTwDuAf5U0LiIurVO/bwPHAlfk9zkM+E+2vPv5ZuDPgPn5fF4LfBS4TdKB\nEbEB+HGuw8eBLwBL8rEP57qeBMwDFgCfBsYCZwB3SDrEY1TNbDgowr07ZtbdJB1GGoN5bER8r2T/\nXOBk4MKIOLfsmF8Cj0TEu0r2bQ88BPw6IqbnfZcD04GDIuK5krJX5/2viojfV6nXIcAi4JKIOKtk\n/7eAU4DzI+KC/s8ufx9JbwHuAU6KiKvyvveR7p5OjYiFJWV3AB4F/j0izijZ/wrgV3n/R2qE0cxs\nq3C3vplZfd8s/UXS/wImAPMl7dq/AeOAW4ApJcXfC1wPjC4rexPQA0ys8bnTSXdILyvb/3XS2NE/\nKk1MJW0jaRdgGfBcnc/o9xe5Pt8pq2eQ7thOHcR7mJk1zd36Zma1vRQRq8r2Tcg/r6hyzCZJPcB2\nwM7A6cDfVigXwCtrfPbepOECy8v2P1JeUNIY4Bzgg8CebE5eg5R01jMhH3NblXrWfVjMzGxrcHJq\nZlZbpS73/l6nvwceqHLc74Dd8r+vJI0dreS/G6/aAN8gdfXPJnXl95GSyn9ncL1ko3L5E9k85rbU\nS1unmmZmtTk5NTMbuqX559qIuLVaIUlPAWuB0bXK1fAbUtK4T8lnwuY7t6XeB8yLiE+XfP72pDu3\npao9aLCUdOf0qQbrama2VXjMqZnZ0C0iJXNn5QeJBpC0G0BEbAKuBd4n6U+qlauhl5QwfrRs/8fZ\nMsncyJbf6Z8ARpftW5ffszxp7QWeB87JK2YNta5mZluF75yamQ1RRISk00hTSf0iP9X/GGms51RS\nl/oxufj/AQ4H7pX0/0hP8+8CTAKOYHPXf6XPWSzpWuBTOTm8hzSVVP+d09IE9YfASXku1oeAycCR\npKmrSv2MlMieLWln0rCFWyLiaUlnkMbRLpb0HeAp0spZ7wLuJCW7ZmYt5eTUzCyp1t1dcX9E/FjS\nZOAfgDOBHYHVpCfb/7mk3JN5SqfPAe8hzRu6BvgFaS7Rek4CfgvMyMffAhwP/BLYUFLuE6RxoR8A\nxpCSyaNId0T/eA4R8YSkvwU+A1xOurM6FVgYEfMlPUZKqM8Cticl3XcAnb6sq5kVhOc5NTMbYfJU\nVouBEyJifrvrY2a2NXnMqZlZgeUposp9itQ1v7DCa2ZmI5q79c3Miu3TkiaR5h99CXgnMA3454h4\nrK01MzNrgZbdOZX0cklXSeqT9Kykyys91Vrj+G9K2iTJA/DNrJvdBbwcOBf4R2B/YBbwsXZWysys\nVVp55/RqYHfS06LbAfNIDwmcWO9ASe8BDiUNxDcz61oRcTNwc7vrYWY2XFpy51TSG0jdTh+KiP+K\niLtI8/IdL2l8nWP3BP6J9MSpVyQxMzMz6yKtunM6GXiONEH1/yatDf090nQmhwI/KD8gT/r8RdJk\n09uR7hTslLeqJO1KSoRXMHBaFTMzMzMrhjHAa4HeiFhTq2CrktPxpATzAAZ267+UX6tkLGkOv18D\n7yeNsVoIfBj4Qo3PmgZctTUqbWZmZmYtdQJp6GdVQ0pOJV0InF2jSJAS0lcAO5C69e/Px34cWED1\nO6ETgHHAlIhYnY9ZA7xa0l4RsarKcSsArrzySg444ICKBWbOnMns2bNrVNtqcfwa59g1x/FrnGPX\nHMevcY5dczo1fg8//DAnnngi5LytlqHeOf1H6q8SsgzoIa3wd7+kM0krjfTfMa2cQcLbSEntKknl\na0HfAexT5bgNAP/yL/9CT0/PgBdmzJjBjBkz6OnpYeLEiXWqbdU4fo1z7Jrj+DXOsWuO49c4x645\nnRC/+fPnM3/+wPVB+vr6+v9ZdwjmkJLTPEag5jgBAEl96YfOBs4HTge2JS2Vd7ykT0dE+XrPV5CW\n8/sB8J3883ukpfXOr/eZs2fPHvEX08zMzGyk6785WGrx4sVMmjRpUMcP6Wl9SRfmuUerbRslvQ54\nClhHWnP6+8BS0hrSG4DfA3+T32+JpGMAIuJZ4C9y2b8FTiOtgPIz4Lih1NPMzMzMRqZWdeuvBv4A\n7AwcDUwHrgVOIa0HPTmXnUAaAtDvz4BbgP8PvBr4Lelu6rvqVWzmzJlVu/XNzMzMbHjU6davq1Xd\n+nezOemcAxwLnJQ/bw0p8SQiyseW7g9MJE03sB54EHgYmCFp+4j4fbXPrNWt7wS1OY5f4xy75jh+\njXPsmuP4Nc6xa04nxK/Zbn1FRCvqhaRbgCNI00d9mbQy1IukxPTBiHizpCXA2RHxgzzP6Qu5/KnA\nctJE/GeShh+MrZScSpoILFq0aJHHnJqZmZkVUElyOikiFtcq28rlS99PGnsKaQ3o7wKfzPtelveX\nduvvSXpoahs2z3+lvP2h1l1TcLe+mZmZWRE0263fyjun25IefnqQNH/peOAB4GBgaUQcVOGYLwPv\niIiDJb0VuJ101/ThSuXzMb5zamZmZlZgQ7lzOqSn9Ydot/zzT4GbgPeSJuAfQ+re73/6/9slx3wT\n2FfSbNLd0zW5jp9rpiLl2bsNjePXOMeuOY5f4xy75jh+jXPsmuP4tTY5hdQlv4i0xOj3gD7SuNJd\n8uvjyQ9HAUTECtKT+acAewGvBJ6IiO/V+6CZM2dy9NFHD9j6L7AvdHMcv8Y5ds1x/Brn2DXH8Wuc\nY9ecTojf/Pnzt8jJZs6cOejjWznm9Ln8c0FEnNu/U9IjwFiAiDi1wnH7ke6YbkeaI/VPBvNhnoTf\nzMzMrP2GdRL+Ido5/9y/f4ckke6GvljpAEkTgEtID0cdDzzSwvqZmZmZWcG0uls/gHdLOlnSG0hj\nSrcFnoSBY04ljQJuJD089XfAT4EdgW0k7dTiepqZmZlZAbSyW/9p0vKjc4ELgN1JS5HeTJrLFAaO\nOR1H6tIHuCxv/Z6VdGRE3F7hc8YAnHbaaYwbN27AC9OmTWP69On09fWxeHHNB8OsBsevcY5dcxy/\nxjl2zXH8GufYNacT4rdgwQJ6e3sH7Fu7dm3/P8fUO75lU0kBSLoHuDciPpl/F7ASuDQiLiorK+CA\nsrc4E5gKvA9YERHrK3zGB4CrWlB9MzMzM9u6ToiIq2sVaOWdU0jjR+dJWgTcB8wkPQw1D1K3PrBH\nRJwSKUt+qPRgSU8CGyLi4Rqf0QucAKwANmztEzAzMzOzpo0BXkvK22pqaXIaEddI2o2B3frTIqJ/\n5agBU0k1+Blr2LyilJmZmZkV012DKdTSbn0zMzMzs6Fo9dP6ZmZmZmaD5uTUzMzMzArDyamZmZmZ\nFUbHJ6eSzpS0XNJ6SfdIenO761R0kmZJ2lS2PVT/yO4k6e2SrpP0WI7V0RXKXCDpcUkvSPqRpP0r\nvVe3qRc7SXMrtMUb2lXfIpH0GUn3SXpe0hOSvi/pdRXKue1VMJj4uf1VJukjkh6Q1Je3uyRNLyvj\ndldFvfi53XV4cirp/cDFwCzgEOABoDfPIGC1PUiaYWF83t7W3uoU2g6kmSg+SloVbQBJZwMfA04H\n3gKsI7XD7YazkgVVM3bZjQxsizOqlOs2bwe+DhwKHEVafe8mSS/rL+C2V1Pd+GVuf1t6FDgbmAhM\nAm4FrpN0ILjdDULN+GVd3e46+mn9KosAPEpaBOCrba1cgUmaBRwTERPbXZeRRtIm4N0RcV3JvseB\niyJidv59J+AJ4JSIuKY9NS2eKrGbC/RExHvbV7ORIf/R/SQwJSLuzPvc9gapSvzc/gZJ0hrgrIiY\n63Y3dGXx6/p217F3TiVtS/qL5Jb+fXmi/5uBye2q1wgyIXe1LpV0paSm5qPtVpL2If3VW9oOnwfu\nxe1wsA7P3a5LJM2RtEu7K1RQO5PuPj8DbnsNGBC/Em5/NUgaJel4YHtgodvd0JTHr+Slrm53rV4h\nqp12A0aT/lor9QTw+uGvzohyD/BB4JfAq4DzSF86B0XEujbWayQaT/ofXqV2OH74qzPi3AhcCywH\n9gMuBG6QNDk6udtniHKv0NeAOyOif3y4294gVYkfuP1VJekg4G7Sqj8vAMdFxFJJk3G7q6ta/PLL\nXd/uOjk5tQZFROnSYg9Kug/4DXAcMLc9tbJuVNYF+AtJPweWAocDt7WlUsU0BzgQeGu7KzJCVYyf\n219NS4CDgR7gWOA7kg5rb5VGlIrxi4j73e46uFsfeBrYSBpQXGp3YPXwV2fkiog+4FeAn7YcutWA\ncDvcKiJiOem/bbfFTNI3gHcCh0fEb0tectsbhBrx24Lb32YR8VJELMvJ1GdJ3fZn4HY3KDXiV6ls\n17W7jk1OI+JFYBFwZP++3HVzJINc29USSTuS/qOo+cVtW8pfKqsZ2A53Ij0h7HY4RJL2AnbFbRH4\nY2J1DDA1IlaWvua2V1+t+FUp7/ZX3ShgtNtdw0aRhiJuoRvbXad3618CzJO0CLgPmAmMBea1s1JF\nJ+ki4HpSV/6ewPnAi8D8dtarqCTtQErelXftK+lg4JmIeJQ0lu1cSY8AK4DPA6uAH7ShuoVSK3Z5\nm0Uae7U6l/sK6S5+75bv1l0kzSFNL3M0sE5S/52qvojYkP/ttldFvfjltun2V4GkL5HGRa4ExgEn\nAFOAL+Qibnc11Iqf210WER29keZPXAGsJw0+flO761T0jZSErsoxWwlcDezT7noVdQMOAzaRhpGU\nbt8qKXMe8Dhp4HsvsH+7612ErVbsSA8KLCB9QW8AlgGXAa9od72LsFWJ20bg5LJybnsNxM/tr2bs\nLs/xWJ/jcxNwRFkZt7sG4ud2l7aOnufUzMzMzEaWjh1zamZmZmYjj5NTMzMzMysMJ6dmZmZmVhhO\nTs3MzMysMJycmpmZmVlhODk1MzMzs8JwcmpmZmZmheHk1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwn\np2ZmZmZWGE5OzczMzKwwnJyamZmZWWE4OTUzMzOzwnByamZmZmaF4eTUzMzMzArDyamZmZmZFYaT\nUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZmZlZYTg5NTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJ\nqZmZmZkVhpNTMzMzMysMJ6dmZmZmVhhOTs3MzMysMJycmpmZmVlhODk1MzMzs8JwcmpmZmZmheHk\n1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwnp2ZmZmZWGE5OzczMzKwwnJyamZmZWWE4OTUzMzOzwnBy\namZmZmaF4eTUzMzMzArDyamZmZmZFYaTUzMzMzMrDCenZmZmZlYYTk7NzMzMrDCcnJqZmZlZYTg5\nNTMzM7PCcHJqZmZmZoXh5NTMzMzMCsPJqZmZmZkVhpNTMzMzMysMJ6dmZmZmVhhOTs3MzMysMJyc\nmplVIOkwSZskTSnZN0/S8nbWy8ys0zk5NTOrLir8Xr6vI0iaIemT7a6HmZmTUzMzA/gA4OTUzNrO\nyamZmZmZFYaTUzPrKpJeI2mOpCWSXpD0tKRrJO3d4PuNlXSxpJWSNuT3/fsqZU+UdK+kdZKekfRj\nSUc1+LlvzMe/IOlRSZ+VdGoeJ/uaknJHS/qhpMdy/R6RdK6kUSVlbgPeBeydj98kaVnJ69tJOl/S\nr/N7rJT0FUnbNVJ3M7Natml3BczMhtmbgT8D5gOrgNcCHwVuk3RgRGwY4vtdDxwGXA48AEwDLpK0\nR0T8MUmVNAuYBfwE+AfgD8ChwBHAzUP5QEl7ALcBG4EvAi8Ap+X3LB8T+0FgLXAx8Lv8eRcA44Cz\nc5kvAD3AnsCnAOWySFI+xz8H/hlYAvwpMBOYALx3KHU3M6tHER05tt/MrCJJ20fE78v2vQW4Bzgp\nIq7K+w4DbgWmRsTCvG8ucFhE7Jt/Pwb4PnBORHy55P2uISVtEyJiuaT9gF8C34+Iv94K53ApKaE+\nJCJ+nvftDDwCvBzYJyJW1jjfy4ATgV0i4sW873rgT/rPraTsicBcYEpE3F2y/3TgMuCtEXFPs+dk\nZtbP3fpm1lVKEzVJ20jaBVgGPAdMHOLbvQN4Cfh62f6LSd+v78i/v4d0N/KCRupcwTTg7v7EFCAi\nngOuKi9Ydr47StoVuBMYC7xhEJ91LPAw8CtJu/ZvpDu3AqY2dSZmZmXcrW9mXUXSGOAcUnf3nqQE\nC1J3eM8Q325v4PGIWFe2/+GS1wH2BTaV7G/W3sBdFfY/Ur5D0oGkrv+pwE4lLw32fCeQktinKrwW\nwCsH8R5mZoPm5NTMus03gFOA2aSu/D5SkvXvdFhvkqQeYCHprvC5pDvEG4BJwJcZ3PmOAn5OGmOq\nCq8/ulUqa2aWOTk1s27zPmBeRHy6f4ek7YGdG3iv3wBHStqh7O7pAfnnivxzKSnJOxD47wY+p9Ln\n7l9h/4Sy3w8njUE9JiJ+0r8zj4EtV+0BhKXAGyPitgbqaWY2ZB11l8DMbBA2suV33yeA0Q281w2k\nP/I/VrZ/Jqkbf0H+/T9Iyd/n8tPvFUkaL+n1kurVpReYLOmNJcfuQppIv9RG0t3O0mmjtiM9TFVu\nHZW7+a8B9pL04Qr1HSNpbJ26mpkNie+cmlm3+SFwkqTngYeAycCRwNMVylZNJLPrSQ8GfVHSPmye\nSuqvgNkRsRwgIpZK+iKpa/0OSd8Dfk+a1uqxiPhsfr8vAyeTprdaWeNzv0p62v5mSV8nJZanke6o\nvpzNd0HvAp4FrshP+JOPq3SXdBFwnKSLgZ8Cv4uIHwL/BhwHXCZpKmkqrNGku8N/DfwlsLhOnMzM\nBs3JqZl1m0+QnrD/ADCG9OT6UaS7keVJW6Uk7o/7IiIk/RXpKfz3kx6yWgGcFRGzBxwUMStPbP9x\n0ryiL5C6+K8oe+9N9U4gIlZJOhy4FPgMKbG+jDQ36ddI40qJiGckvYs0e8DnSYnqv5GmyOote9s5\nwMH5HD5FSnR/mM/xGNLd4JOBd+e6LyON2/1VvfqamQ2F5zk1M+sQkr4GfBjYMfzlbmYjlMecmpmN\nQHlKrNLfdyV12d/hxNTMRrKWJaeSXi7pKkl9kp6VdLmkHYZw/Dfz+s6faFUdzcxGsLslzZZ0uqTP\nkcaMjiN135uZjVitHHN6NbA76UGD7YB5pHWZT6x3oKT3kNacfqyF9TMzG8n+k7R604dJY1UXAaeW\nThllZjYStWTMqaQ3kJ6CnRQR9+d900hfpntFxOoax+4J3E164vUG0hOvl1Yrb2ZmZmado1Xd+pNJ\nK5Kc1d+tT5qKJEh3RLeQ17j+CrCEdMf1ZmA3Bi63Z2ZmZmYdrFXd+uNJXfkHMLBb/6X8WiVjgfcA\nvyZNyfJy0rJ7HyZNu1JRfghgGmn6lg1bo/JmZmZmtlWNIc3h3BsRa2oVHFJyKulC4OwaRYKUkL4C\n2AH4UEm3/sdJq6VUuxM6gTSYf0p/t7+kNcCrJe0VEauqHDcNuGoo52FmZmZmbXEC6bmkqoZ65/Qf\ngbl1yiwjLYEX/Ylpdmv+WW396reRktpHS1b3G01aoeUOYJ8qx60AOOSQQxg3btyAF6ZNm8b06dOZ\nOXMms2fPrnSsDYLj1zjHrjmOX+Mcu+Y4fo1z7JrTCfFbsGABvb0D1/lYu3Yt999/P+S8rZYhJaf5\nNmzNW7EAkvrSDx0C/DlwFrBHfnnHKoddAfyItJzfp4H9c/1WkZbHq2YDwOWXX87EiRMrFujp6an6\nmtXn+DXOsWuO49c4x645jl/jHLvmdEL8Jk6cyDnnnDNg3+LFi5k0aRIMYgjmkB6IknRhnnu02rZR\n0uuAp0hrPX8XuAS4Engc2AicJmm3/H5L8rJ4RMSzpCXx/i/pKf07SWNU9yCNUTAzMzOzDteqbv3V\nwB/YPL70Y8C1wCnAeuBvgK+Sxpn2lBx7Rj5+f2AX0jyna0hrOv+o1ofOnDmTnp6eAftmzJjBjBkz\n6p2TmZmZmW0l8+fPZ/78+QP29fX1Dfr4VnXr301KOjcB/0p6aOmk/Hn3k6aaIiJGlx36LtLY0gNI\nSeyDwE+A0+p95uzZs0f8bXAzMzOzka7SzcGSbv26WjLPaUQsAW4nJaMfYnO3/q9JDz7tBQO79SVt\nA7yO9ADUicBRwH3Ap4CdJG3faH1897Q5jl/jHLvmOH6Nc+ya4/g1zrFrjuPXohWiACS9njSh/ots\nHn/6SVKX/7MRsbekjaTl9q6QtDfpCa7IG6REVfn3l0XE7yt8zkRg0ZQpU9ytb2ZmZtZm1br1Fy5c\nCGn10MW1jm9lcvoy0gNOC4A3kCbffwDYExgdEXtUOObHwKKI+DtJbyXdfRXwh4gYW+VzJgKLFi1a\n5G59MzMzswIq6davm5y2avlS2Dyf6V8Cs4BDSMnpXqSn9iu5GzhSUg/wbWBpruNPmqlIefZuQ+P4\nNc6xa47j1zjHrjmOX+Mcu+Y4fq1NTvtVvTWbp6b6dsmubwL7kpLUZcB+pDunX2mmAr7QzXH8GufY\nNcfxa5xj1xzHr3GOXXMcv6FPJTUUz+WfNwEXALsDPyNNqt//lP544NX9B0TECknfIE0ddQCwFngm\nIm6u92GeSsrMzMys/YZ1Kqkh6u/Wfz4iXgtpySigD3gWICJOLT1A0gTgI6QpqP4KmAQcM5gP81RS\nZmZmZu1XyKmkSgTwbkknS3oDqdt+W+BJGNitL2kUcCMwDvg74KekpU63kbRTpTc3MzMzs87Syjun\nT5MefJrLwG79m0nLksLAbv1xpDGmAJflrd+zko6MiNsrfM4YgNNOO41x48YNeGHatGlMnz6dvr4+\nFi+u+WCY1eD4Nc6xa47j1zjHrjmOX+Mcu+Z0QvwWLFhAb2/vgH1r167t/+eYese3bCopAEn3APdG\nxCfz7wJWApdGxEVlZUUaZ1rqTGAq8D5gRUSsr/AZHwCuakH1zczMzGzrOiEirq5VoJV3TgEuAeZJ\nWkRa7WkmMBaYB6lbH9gjIk6JlCU/VHqwpCeBDRHxcI3P6AVOIE3gv2Frn4CZmZmZNW0M8FpS3lZT\nS5PTiLhG0m4M7NafFhFP5SIDntZv8DPWADUzcDMzMzNru7sGU6il3fpmZmZmZkMxHJPwm5mZmZkN\nipNTMzMzMysMJ6dmZmZmVhgdn5xKOlPScknrJd0j6c3trlPRSZolaVPZ9lD9I7uTpLdLuk7SYzlW\nR1coc4GkxyW9IOlHkvZvR12Lpl7sJM2t0BZvaFd9i0TSZyTdJ+l5SU9I+r6k11Uo57ZXwWDi5/ZX\nmaSPSHpAUl/e7pI0vayM210V9eLndtfhyamk9wMXA7OAQ4AHgN48g4DV9iBphoXxeXtbe6tTaDuQ\nZqL4KGlVtAEknQ18DDgdeAuwjtQOtxvOShZUzdhlNzKwLc6oUq7bvB34OnAocBRp9b2bJL2sv4Db\nXk1145e5/W3pUeBsYCJpmfFbgeskHQhud4NQM35ZV7e7jn5av8oiAI+SFgH4alsrV2CSZgHHRMTE\ndtdlpJG0CXh3RFxXsu9x4KKImJ1/3wl4AjglIq5pT02Lp0rs5gI9EfHe9tVsZMh/dD8JTImIO/M+\nt71BqhI/t79BkrQGOCsi5rrdDV1Z/Lq+3XXsnVNJ25L+Irmlf1+e6P9mYHK76jWCTMhdrUslXSmp\nqflou5WkfUh/9Za2w+eBe3E7HKzDc7frEklzJO3S7goV1M6ku8/PgNteAwbEr4TbXw2SRkk6Htge\nWOh2NzTl8St5qavbXatXiGqn3YDRpL/WSj0BvH74qzOi3AN8EPgl8CrgPNKXzkERsa6N9RqJxpP+\nh1epHY4f/uqMODcC1wLLgf2AC4EbJE2OTu72GaLcK/Q14M6I6B8f7rY3SFXiB25/VUk6CLibtOrP\nC8BxEbFU0mTc7uqqFr/8cte3u05OTq1BEVG6tNiDku4DfgMcB8xtT62sG5V1Af5C0s+BpcDhwG1t\nqVQxzQEOBN7a7oqMUBXj5/ZX0xLgYKAHOBb4jqTD2lulEaVi/CLifre7Du7WB54GNpIGFJfaHVg9\n/NUZuSKiD/gV4Kcth241INwOt4qIWE76b9ttMZP0DeCdwOER8duSl9z2BqFG/Lbg9rdZRLwUEcty\nMvVZUrf9GbjdDUqN+FUq23XtrmOT04h4EVgEHNm/L3fdHMkg13a1RNKOpP8oan5x25byl8pqBrbD\nnUhPCLsdDpGkvYBdcVsE/phYHQNMjYiVpa+57dVXK35Vyrv9VTcKGO1217BRpKGIW+jGdtfp3fqX\nAPMkLQLuA2YCY4F57axU0Um6CLie1JW/J3A+8CIwv531KipJO5CSd+Vd+0o6GHgmIh4ljWU7V9Ij\nwArg88Aq4AdtqG6h1Ipd3maRxl6tzuW+QrqL37vlu3UXSXNI08scDayT1H+nqi8iNuR/u+1VUS9+\nuW26/VUg6UukcZErgXHACcAU4Au5iNtdDbXi53aXRURHb6T5E1cA60mDj9/U7joVfSMloatyzFYC\nVwP7tLteRd2Aw4BNpGEkpdu3SsqcBzxOGvjeC+zf7noXYasVO9KDAgtIX9AbgGXAZcAr2l3vSQsY\nWQAAAJpJREFUImxV4rYROLmsnNteA/Fz+6sZu8tzPNbn+NwEHFFWxu2ugfi53aWto+c5NTMzM7OR\npWPHnJqZmZnZyOPk1MzMzMwKw8mpmZmZmRWGk1MzMzMzKwwnp2ZmZmZWGE5OzczMzKwwnJyamZmZ\nWWE4OTUzMzOzwnByamZmZmaF4eTUzMzMzArDyamZmZmZFYaTUzMzMzMrjP8BAxS2H/AO2kQAAAAA\nSUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "plt.figure(figsize=[8,8])\n", + "\n", + "plt.subplot(611) ; plt.title('y')\n", + "plt.imshow(_y.T, interpolation='none')\n", + "plt.subplot(612) ; plt.title('y_hat')\n", + "plt.imshow(_y_hat[0,:,:].T, interpolation='none')\n", + "plt.subplot(613) ; plt.title('w_W')\n", + "plt.imshow(_w_w[0,:,:].T, interpolation='none')\n", + "plt.subplot(614) ; plt.title('w_r')\n", + "plt.imshow(_w_r[0,:,:,0].T, interpolation='none')\n", + "plt.subplot(615) ; plt.title('free gate')\n", + "plt.imshow(_f[0,:,:].T, interpolation='none')\n", + "plt.subplot(616) ; plt.title('alloc. gate')\n", + "plt.imshow(_g_a[0,:,:].T, interpolation='none')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/copy/.ipynb_checkpoints/visualization-checkpoint.ipynb b/repeat-copy/.ipynb_checkpoints/visualization-checkpoint.ipynb similarity index 100% rename from copy/.ipynb_checkpoints/visualization-checkpoint.ipynb rename to repeat-copy/.ipynb_checkpoints/visualization-checkpoint.ipynb diff --git a/repeat-copy/models/checkpoint b/repeat-copy/models/checkpoint new file mode 100644 index 0000000..ac2497e --- /dev/null +++ b/repeat-copy/models/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "model.ckpt-5000" +all_model_checkpoint_paths: "model.ckpt-5000" diff --git a/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 new file mode 100644 index 0000000..63af8c5 Binary files /dev/null and b/repeat-copy/models/model.ckpt-1000.data-00000-of-00001 differ diff --git a/repeat-copy/models/model.ckpt-1000.index b/repeat-copy/models/model.ckpt-1000.index new file mode 100644 index 0000000..9c25af7 Binary files /dev/null and b/repeat-copy/models/model.ckpt-1000.index differ diff --git a/repeat-copy/models/model.ckpt-1000.meta b/repeat-copy/models/model.ckpt-1000.meta new file mode 100644 index 0000000..77dbcc0 Binary files /dev/null and b/repeat-copy/models/model.ckpt-1000.meta differ diff --git a/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 new file mode 100644 index 0000000..577a6ba Binary files /dev/null and b/repeat-copy/models/model.ckpt-2000.data-00000-of-00001 differ diff --git a/repeat-copy/models/model.ckpt-2000.index b/repeat-copy/models/model.ckpt-2000.index new file mode 100644 index 0000000..7f94a9c Binary files /dev/null and b/repeat-copy/models/model.ckpt-2000.index differ diff --git a/repeat-copy/models/model.ckpt-2000.meta b/repeat-copy/models/model.ckpt-2000.meta new file mode 100644 index 0000000..77dbcc0 Binary files /dev/null and b/repeat-copy/models/model.ckpt-2000.meta differ diff --git a/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 new file mode 100644 index 0000000..e423541 Binary files /dev/null and b/repeat-copy/models/model.ckpt-3000.data-00000-of-00001 differ diff --git a/repeat-copy/models/model.ckpt-3000.index b/repeat-copy/models/model.ckpt-3000.index new file mode 100644 index 0000000..10b5931 Binary files /dev/null and b/repeat-copy/models/model.ckpt-3000.index differ diff --git a/repeat-copy/models/model.ckpt-3000.meta b/repeat-copy/models/model.ckpt-3000.meta new file mode 100644 index 0000000..ea8a258 Binary files /dev/null and b/repeat-copy/models/model.ckpt-3000.meta differ diff --git a/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 new file mode 100644 index 0000000..bb7e770 Binary files /dev/null and b/repeat-copy/models/model.ckpt-4000.data-00000-of-00001 differ diff --git a/repeat-copy/models/model.ckpt-4000.index b/repeat-copy/models/model.ckpt-4000.index new file mode 100644 index 0000000..32b887c Binary files /dev/null and b/repeat-copy/models/model.ckpt-4000.index differ diff --git a/repeat-copy/models/model.ckpt-4000.meta b/repeat-copy/models/model.ckpt-4000.meta new file mode 100644 index 0000000..ea8a258 Binary files /dev/null and b/repeat-copy/models/model.ckpt-4000.meta differ diff --git a/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 b/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 new file mode 100644 index 0000000..0ee35fb Binary files /dev/null and b/repeat-copy/models/model.ckpt-5000.data-00000-of-00001 differ diff --git a/repeat-copy/models/model.ckpt-5000.index b/repeat-copy/models/model.ckpt-5000.index new file mode 100644 index 0000000..b0059a3 Binary files /dev/null and b/repeat-copy/models/model.ckpt-5000.index differ diff --git a/repeat-copy/models/model.ckpt-5000.meta b/repeat-copy/models/model.ckpt-5000.meta new file mode 100644 index 0000000..ea8a258 Binary files /dev/null and b/repeat-copy/models/model.ckpt-5000.meta differ diff --git a/copy/nn_controller.py b/repeat-copy/nn_controller.py similarity index 66% rename from copy/nn_controller.py rename to repeat-copy/nn_controller.py index 7ecec8a..f2c8111 100755 --- a/copy/nn_controller.py +++ b/repeat-copy/nn_controller.py @@ -11,19 +11,21 @@ A 2-Layer feedforward neural network with 128, 256 nodes respectively class NNController(Controller): def init_controller_params(self): + h1_dim = 128 + h2_dim = 256 init = tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32) - self.params['W1'] = tf.get_variable("W1", [self.chi_dim, 128], initializer=init) - self.params['b1'] = tf.get_variable("b1", [128], initializer=init) - self.params['W2'] = tf.get_variable("W2", [128, 256], initializer=init) - self.params['b2'] = tf.get_variable("b2", [256], initializer=init) + self.params['W1'] = tf.get_variable("W1", [self.chi_dim, h1_dim], initializer=init) + self.params['b1'] = tf.get_variable("b1", [h1_dim], initializer=init) + self.params['W2'] = tf.get_variable("W2", [h1_dim, h2_dim], initializer=init) + self.params['b2'] = tf.get_variable("b2", [h2_dim], initializer=init) def nn_step(self, X, state): z1 = tf.matmul(X, self.params['W1']) + self.params['b1'] - h1 = tf.nn.relu(z1) + h1 = tf.nn.elu(z1) z2 = tf.matmul(h1, self.params['W2']) + self.params['b2'] - h2 = tf.nn.relu(z2) + h2 = tf.nn.elu(z2) return h2, state def zero_state(self): diff --git a/repeat-copy/nn_controller.pyc b/repeat-copy/nn_controller.pyc new file mode 100644 index 0000000..da35d7f Binary files /dev/null and b/repeat-copy/nn_controller.pyc differ diff --git a/repeat-copy/repeat-copy.ipynb b/repeat-copy/repeat-copy.ipynb new file mode 100644 index 0000000..11a87a5 --- /dev/null +++ b/repeat-copy/repeat-copy.ipynb @@ -0,0 +1,339 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Repeat Copy Task\n", + "### Using a Differentiable Memory Architecture (DNC)\n", + "\n", + "\"DNC\n", + "\n", + "**Sam Greydanus $\\cdot$ February 2017 $\\cdot$ MIT License.**\n", + "\n", + "Represents the state of the art in differentiable memory. Inspired by this [Nature paper](http://www.nature.com/nature/journal/v538/n7626/full/nature20101.html). Some ideas taken from [this Gihub repo](https://github.com/Mostafa-Samir/DNC-tensorflow)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "import sys\n", + "sys.path.insert(0, '../dnc')\n", + "\n", + "from dnc import DNC\n", + "from nn_controller import NNController\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xydim = 6\n", + "tf.app.flags.DEFINE_integer(\"xlen\", xydim, \"Input dimension\")\n", + "tf.app.flags.DEFINE_integer(\"ylen\", xydim, \"output dimension\")\n", + "tf.app.flags.DEFINE_integer(\"length\", 5, \"Sequence length\")\n", + "tf.app.flags.DEFINE_integer(\"reps\", 3, \"Number of repeats for copy task\")\n", + "tf.app.flags.DEFINE_integer(\"batch_size\", 1, \"Size of batch in minibatch gradient descent\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"R\", 1, \"Number of DNC read heads\")\n", + "tf.app.flags.DEFINE_integer(\"W\", 10, \"Word length for DNC memory\")\n", + "tf.app.flags.DEFINE_integer(\"N\", 7, \"Number of words the DNC memory can store\")\n", + "\n", + "tf.app.flags.DEFINE_integer(\"print_every\", 100, \"Print training info after this number of train steps\")\n", + "tf.app.flags.DEFINE_integer(\"iterations\", 40000, \"Number of training iterations\")\n", + "tf.app.flags.DEFINE_float(\"lr\", 1e-4, \"Learning rate (alpha) for the model\")\n", + "tf.app.flags.DEFINE_float(\"momentum\", .9, \"RMSProp momentum\")\n", + "tf.app.flags.DEFINE_integer(\"save_every\", 1000, \"Save model after this number of train steps\")\n", + "tf.app.flags.DEFINE_string(\"save_path\", \"models/model.ckpt\", \"Where to save checkpoints\")\n", + "FLAGS = tf.app.flags.FLAGS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data functions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAABwCAYAAAAKXJmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAADxVJREFUeJzt3W2sHNV5wPH/c12wEwhGxSlYJQJSkja+jmhsh5QmvDQ2\nMqIqtFJFcKkqXiIEJP3gqi1BIJFSStJGBFqIq6oNOBXh0lRtFFADBgNtigilsRNaX0NReAm02BSD\ncCRju9j39MOsyX3be/fsndndmfv/Sfvh7p6dfc48Z+fc2Zl5JlJKSJKk+hrqdwCSJGlunMwlSao5\nJ3NJkmrOyVySpJpzMpckqeaczCVJqjknc0mSas7JXJKkmnMylySp5n6qyoVHxDHAWuBFYF+VnyVJ\nUsMsAk4ENqWUXp+pYVeTeUR8Bvh94DjgKeB3U0r/Pk3TtcDXu/kMSZIEwEXA3TM1yJ7MI+JTwM3A\n5cCTwHpgU0R8MKW0a1LzFwHOv+vXWPKhJRNeeGj9Zs6+ZU3ux89dZi36oVUbs9qPbblk2uf71t+K\ntVud7fp7x6rOh9xlWw50G1ZnMsfCgjZj4QHgnOkWnxnO2Pcuzmpf1tjM1S63ubd5uGPVYZmfnPcB\nl36v8/HTflxuotgnmajysdknTd1OtTPo/d319C6+9dv3QWsunUk3e+brgb9KKf0tQERcAfwqcCnw\nZ5Pa7gNY8qElLF1x3IQXFi5eOOW5nsidzDMXP9amT33rb8Xarc5FR7frb+cb8KUr3u4uqE7lTuZt\nnl8ELJ1u8ZnhtBs77ZQ1NnO1G8v592w6PLN93gfkjZ9243L67FY+NvukqdupdmrU31kPU2dtDyLi\nMGAl8PCh51Jx27XNwGm50UmSpLnL/ed+CcUOyquTnn+V4vi5JEnqMS9NkySp5nKPme8CDgLHTnr+\nWGBnuzc9tH4zCxcvnPDc4hOOyvzoehtet6zfIfTU8IXzp7/L+x1Aj82vsTy/sju/cjtY/d02Msro\nyPYJz+3fvb/j92dN5imltyNiC7AauBcgIqL191+0e9/Zt6ypy0kGlVm+brjfIfTU8Dzq74f7HUCP\nza+xPL8m8/mV28Hq7/J1w1Pi2bF1J19deWdH7+/mbPYvAxtbk/qhS9PeDWzsYlmSJGmOsifzlNI3\nImIJcAPFz+s/ANamlF4rOzhJkjS7rirApZQ2ABtKjkWSJHXBs9klSaq5SJllmyLidOAPKIrHLAV+\nPaV0b5u2K4Atl225pL4nwOWWtYqoJo6GyFmdA7cq80uc5cntcM3H5iCtzspXZW7lyaEv5i0/M56x\ng5/Laj+0IC+esbFrstrnys3Xnwxllg7OXJ/XHsyrCNh5PDuAvwZYmVLaOlPLbvbMj6A4Tn4V+RUr\nJUlSybo5Ae4BintLHLosTZIk9ZHHzCVJqjknc0mSas7JXJKkmuvqOvNc09VmH163bKBK6UmS1D/b\nWo/xZr2N+Tt6Mplbm12SpJksZ+q9AN65NG1W2ZN5RBwBnMxPrsR7f0ScAryRUno5d3mSJGluutkz\nXwU8SnGNeQJubj3/NeDSkuKSJEkd6uY683/BE+ckSRoYTsqSJNVc1p55RFwD/AbwC8Be4HHg6pTS\ns6VFlFvDOLdmcG5N4qprJA9aDeMFeTWMq6tJTPa6vO7g/2W1r3zsDNryM8fawI2dCpefu+zrxvJi\nzy3mnjsWFmSOharHZtVya4/mj7XDM9tnjs0Ox8+OrQe4Y1Vny8zdMz8duA34GLAGOAx4MCLelbkc\nSZJUkqw985TSueP/joiLgf+luIPaY+WFJUmSOjXXY+ZHU5zR/kYJsUiSpC50PZm37ph2K/BYSml7\neSFJkqQcc6kAtwFYBny8pFgkSVIXuprMI+J24Fzg9JTSjtnaW5tdkqT2RkdGGb1n4o/c+97c3/H7\nuynnejtwPnBmSumlTt5jbXZJktobXjfM8KQd3B1bd3LHqjs7en/udeYbgHXAecCeiDi29dLulFLn\nt3eRJEmlyT0B7grgKOCfgVfGPS4oNyxJktSp3OvMLf8qSdKAcXKWJKnmco+ZXwFcCZzYemoUuCGl\n9EBpEVVcw7jq5efWPK5a1TWMq1x+br3jbJm1x7PHzoGK7wMwaDLXZ/bYOVDhfQAGzVDeyjk4VnHt\n9NxkDZjI3G29dizvPg+5Ol2dOas9d8/8ZeBqYAVFCddHgHsjYlnmciRJUklyj5n/06SnrouIKylu\nvGIVOEmS+qDrCnARMURxFvtC4F9Li0iSJGXppmjMcuC7wCLgLeCClNIPyw5MkiR1ppuz2Z8BTgFO\nBW4H7omIj5QalSRJ6lj2nnlK6QDwfOvP70fEqRRnuF/e7j3WZpckqb1tI6OMjkw89Wz/7gprs09j\nCFgwUwNrs0uS1N7ydcNTdnB3bN3JV1dWU5v9JuB+4CXgPcBFwBnAjTnLkSRJ5cndM/8Z4GvAUmA3\n8B/A2pTSo2UHJkmSOpN7nfmnqwpEkiR1x9rskiTV3JxOgIuIzwE3AbemlH6vbcOUikcHcmubZ9dO\njy9ktSezJPFYxTWSO1yN78itb55bm/3GyKx/nbE+rx3Li6XyuvtDeWMn+74Bmaoea1XX9b8xs3Z6\n7vKzlp071jK/iJXX3c/eTl1TTRxdyt6uVV13P3N9Xpc7firQ9Z55RHyU4nK0p8oLR5Ik5epqMo+I\nI4G7gE8Db5YakSRJytLtnvlXgPtSSo+UGYwkScrXTW32C4FfBFaVH44kScqVWzTmeOBWYE1Kqf9H\n/CVJUvae+UrgvcDWiHfOdV0AnBERnwUWpjT1vMSH1j/MwqMn1Wa/cBnL1y3rImRJkpql17XZNwMf\nnvTcRuBp4IvTTeQAZ9+y2trskiS10dPa7CmlPcCEfx0iYg/wekrp6ZxlSZKkcpRRAS7zcn9JklSm\nOd8CNaX0yTICkSRJ3Yk2h7nLWXjECmDLZVsu6fyYeW48uTUnK+xvN7LL12aWYazz6syNpXJ1XplU\nP9ZyDdLqrHysDdh2p+qxOWjbtapVNX7GHTNfmVLaOlPbrJ/ZI+L6iBib9Ng++zslSVJVuvmZfRuw\nmp+Uoj9QXjiSJClXN5P5gZTSa6VHIkmSutLN2ewfiIj/iYjnIuKuiHhf6VFJkqSO5U7mTwAXA2uB\nK4CTgO9ExBElxyVJkjqUWzRm07g/t0XEk8CPgAuAtmVqHlq/mYWLJ5ZzXXzCUZxz29qcj6+1bSPb\n51X52tGRUYYnVTNqqvmW220jo1MqVTXVfOorzL/+DtJ2qtflXCdIKe2OiGeBk2dqd/Yta6ZcmvZ3\n5/39XD66dkbvmV8b/NF7tg/Ml6Rq8y63I9vnzQZ/PvUV5mF/B2g7NddyrnOqABcRR1JM5DvmshxJ\nktS93OvMvxQRZ0TECRHxy8A3gbeBkUqikyRJs8r9mf144G7gGOA14DHgl1JKr5cdmCRJ6kzuCXDr\nMpe/CGDX07umvLB/9352bN053YfkfcKAlcxsZ/+b0/d3KDOcsenW2Qz6tTr3telvlbFUrk1n2+V2\n0MZmWWOt7Xc30yB91dstu6y+1qX+aFnb5UHbrrVTxnYKqttWjZs7F80aQ8W12X8L+HplHyBJUvNd\nlFK6e6YGVU/mx1Bck/4isK+yD5IkqXkWAScCm2Y7nF3pZC5Jkqo3p0vTJElS/zmZS5JUc07mkiTV\nnJO5JEk115fJPCI+ExEvRMTeiHgiIj7ajziqFhHXR8TYpMf22d85+CLi9Ii4t3U73LGIOG+aNjdE\nxCsR8VZEPBQRM9bwH2Sz9Tci7pwm19/uV7xzERHXRMSTEfHjiHg1Ir4ZER+cpl0j8ttJf5uS34i4\nIiKeiojdrcfjEXHOpDaNyCvM3t+m5BX6MJlHxKeAm4HrgY8ATwGbImJJr2PpkW3AscBxrccn+htO\naY4AfgBcBUy5JCIirgY+C1wOnArsocjz4b0MskQz9rflfibmOrfI0qA4HbgN+BiwBjgMeDAi3nWo\nQcPyO2t/W5qQ35eBq4EVwErgEeDeiFgGjcsrzNLflibkFVJKPX1Q3BP9z8f9HcB/A3/Y61h60Nfr\nga39jqMH/RwDzpv03CvA+nF/HwXsBS7od7wV9fdO4B/7HVtF/V3S6vMn5kl+p+tvk/P7OnBJ0/Pa\npr+NyWtP98wj4jCK/44ePvRcKtboZuC0XsbSQx9o/TT7XETcFRHv63dAVYuIkyj+wx2f5x8D/0Zz\n8wxwVutn2mciYkNE/HS/AyrJ0RS/RrwB8yK/E/o7TqPyGxFDEXEhsBD4TtPzOrm/415qRF7ndD/z\nLiwBFgCvTnr+VeDnexxLLzwBXAz8F7AU+DzFl2Z5SmlPH+Oq2nEUG8Pp8nzc1OaNcD/wD8ALwM8B\nXwC+HRGntf5hraWICOBW4LGU0qHzPRqb3zb9hQblNyKWA9+lqC72FsVe93MRcRoNzGu7/rZebkxe\nez2ZzysppU3j/twWEU8CPwIuoPh5Rw2RUvrGuD9HI+I/geeAs4BH+xJUOTYAy4CP9zuQHpm2vw3L\n7zPAKcBi4DeBeyLizP6GVKlp+5tS+n6T8trrE+B2AQcpTjYY71ighFsTDbaU0m7gWaC2Z4d2aCfF\nuRDzMs8AKaUXKMZ7bXMdEbcD5wJnpZR2jHupkfmdob9T1Dm/KaUDKaXnW5PZtRQ/o19JQ/M6Q3+n\na1vbvPZ0Mk8pvQ1sAVYfeq71s9Zq4PFextIPEXEkxSCZcUNRd60vxE4m5vkoirOFG59ngIg4HjiG\nmua6NbGdD/xKSuml8a81Mb8z9bdN+1rnd5IhYEET89rGEMXh3inqnNd+/Mz+ZWBjRGwBngTWA+8G\nNvYhlkpFxJeA+yh+Wv9Z4I+At4GRfsZVhog4guIfk0N38n1/RJwCvJFSepniuON1EfFDirvm/THF\nVQvf6kO4czZTf1uP6ymOve1stftTil9hNk1d2mCLiA0Ul+ecB+yJiEN7artTSofuftiY/M7W31bu\nG5HfiLiJ4jjxS8B7gIuAM4AbW00ak1eYub9NyivQ+0vTWucUXEUxUPZSnJiwqt+n9VfUzxGKL8Je\nisF0N3BSv+MqqW9nUly+c3DS445xbT5PcanLWxRfjpP7HXcV/aU4seYBig3CPuB54C+B9/Y77i77\nOl0/DwK/M6ldI/I7W3+blF/gb1rx723150Hgk03M62z9bVJeU0reAlWSpLqzNrskSTXnZC5JUs05\nmUuSVHNO5pIk1ZyTuSRJNedkLklSzTmZS5JUc07mkiTVnJO5JEk152QuSVLNOZlLklRzTuaSJNXc\n/wNYM0Jj7W43GwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_sequence(length, reps, dim):\n", + " X = [np.concatenate((np.random.randint(2, size=(length,dim)), np.zeros((length + 3,dim)))) for _ in range(reps)]\n", + " X = np.vstack(X) ; X[:,dim-1] = 0\n", + " \n", + " X = np.concatenate((X[-1:,:],X[:-1,:]))\n", + " y = np.concatenate((X[-(length + 2):,:],X[:-(length + 2),:]))\n", + " markers = range(length+1, X.shape[0], 2*length+3)\n", + " X[markers,dim-1] = 1\n", + " return X, y\n", + " \n", + "def next_batch(batch_size, length, reps, dim):\n", + " X_batch = []\n", + " y_batch = []\n", + " for _ in range(batch_size):\n", + " X, y = get_sequence(length, reps, dim)\n", + " X_batch.append(X) ; y_batch.append(y)\n", + " return [X_batch, y_batch]\n", + "\n", + "batch = next_batch(1, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "plt.imshow(batch[0][0].T - batch[1][0].T, interpolation='none')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def binary_cross_entropy(y_hat, y):\n", + " return tf.reduce_mean(-y*tf.log(y_hat) - (1-y)*tf.log(1-y_hat))\n", + "\n", + "def llprint(message):\n", + " sys.stdout.write(message)\n", + " sys.stdout.flush()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build graph, initialize everything" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "building graph...\n", + "defining loss...\n", + "computing gradients...\n", + "init variables... \n" + ] + } + ], + "source": [ + "sess = tf.InteractiveSession()\n", + "\n", + "llprint(\"building graph...\\n\")\n", + "optimizer = tf.train.RMSPropOptimizer(FLAGS.lr, momentum=FLAGS.momentum)\n", + "dnc = DNC(NNController, FLAGS)\n", + "\n", + "llprint(\"defining loss...\\n\")\n", + "y_hat, outputs = dnc.get_outputs()\n", + "y_hat = tf.clip_by_value(tf.sigmoid(y_hat), 1e-6, 1. - 1e-6)\n", + "loss = binary_cross_entropy(y_hat, dnc.y)\n", + "\n", + "llprint(\"computing gradients...\\n\")\n", + "gradients = optimizer.compute_gradients(loss)\n", + "grad_op = optimizer.apply_gradients(gradients)\n", + "\n", + "llprint(\"init variables... \\n\")\n", + "sess.run(tf.global_variables_initializer())\n", + "llprint(\"ready to train...\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# tf parameter overview\n", + "total_parameters = 0 ; print \"model overview...\"\n", + "for variable in tf.trainable_variables():\n", + " shape = variable.get_shape()\n", + " variable_parameters = 1\n", + " for dim in shape:\n", + " variable_parameters *= dim.value\n", + " print '\\tvariable \"{}\" has {} parameters' \\\n", + " .format(variable.name, variable_parameters)\n", + " total_parameters += variable_parameters\n", + "print \"total of {} parameters\".format(total_parameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "global_step = 0\n", + "saver = tf.train.Saver(tf.global_variables())\n", + "load_was_success = True # yes, I'm being optimistic\n", + "try:\n", + " save_dir = '/'.join(FLAGS.save_path.split('/')[:-1])\n", + " ckpt = tf.train.get_checkpoint_state(save_dir)\n", + " load_path = ckpt.model_checkpoint_path\n", + " saver.restore(sess, load_path)\n", + "except:\n", + " print \"no saved model to load.\"\n", + " load_was_success = False\n", + "else:\n", + " print \"loaded model: {}\".format(load_path)\n", + " saver = tf.train.Saver(tf.global_variables())\n", + " global_step = int(load_path.split('-')[-1]) + 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "loss_history = []\n", + "for i in xrange(global_step, FLAGS.iterations + 1):\n", + " llprint(\"\\rIteration {}/{}\".format(i, FLAGS.iterations))\n", + "\n", + " rlen = np.random.randint(1, FLAGS.length + 1)\n", + " rreps = np.random.randint(1, FLAGS.reps + 1)\n", + " X, y = next_batch(FLAGS.batch_size, rlen, rreps, FLAGS.xlen)\n", + " tsteps = rreps*(2*rlen+3)\n", + "\n", + " fetch = [loss, grad_op]\n", + " feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "\n", + " step_loss, _ = sess.run(fetch, feed_dict=feed)\n", + " loss_history.append(step_loss)\n", + "\n", + " if i % 100 == 0:\n", + " llprint(\"\\n\\tloss: {:03.4f}\\n\".format(np.mean(loss_history)))\n", + " loss_history = []\n", + " if i % FLAGS.save_every == 0 and i is not 0:\n", + " llprint(\"\\n\\tSAVING MODEL\\n\")\n", + " saver.save(sess, FLAGS.save_path, global_step=i)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X, y = next_batch(FLAGS.batch_size, FLAGS.length, FLAGS.reps, FLAGS.xlen)\n", + "tsteps = FLAGS.reps*(2*FLAGS.length+3)\n", + "\n", + "feed = {dnc.X: X, dnc.y: y, dnc.tsteps: tsteps}\n", + "fetch = [outputs['y_hat'], outputs['w_w'], outputs['w_r'], outputs['f'], outputs['g_a']]\n", + "[_y_hat, _w_w, _w_r, _f, _g_a] = sess.run(fetch, feed)\n", + "_y = y[0] ; _X = X[0]\n", + "\n", + "plt.figure(figsize=[8,8])\n", + "\n", + "plt.subplot(611) ; plt.title('y')\n", + "plt.imshow(_y.T, interpolation='none')\n", + "plt.subplot(612) ; plt.title('y_hat')\n", + "plt.imshow(_y_hat[0,:,:].T, interpolation='none')\n", + "plt.subplot(613) ; plt.title('w_W')\n", + "plt.imshow(_w_w[0,:,:].T, interpolation='none')\n", + "plt.subplot(614) ; plt.title('w_r')\n", + "plt.imshow(_w_r[0,:,:,0].T, interpolation='none')\n", + "plt.subplot(615) ; plt.title('free gate')\n", + "plt.imshow(_f[0,:,:].T, interpolation='none')\n", + "plt.subplot(616) ; plt.title('alloc. gate')\n", + "plt.imshow(_g_a[0,:,:].T, interpolation='none')\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/repeat-copy/rnn_controller.py b/repeat-copy/rnn_controller.py new file mode 100755 index 0000000..f35b788 --- /dev/null +++ b/repeat-copy/rnn_controller.py @@ -0,0 +1,26 @@ +import numpy as np +import tensorflow as tf +from controller import Controller + + +""" +A 1-Layer recurrent neural network (LSTM) with 64 hidden nodes +""" + +class RNNController(Controller): + + def init_controller_params(self): + rnn_dim = 64 + init = tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32) + + self.params['cell'] = tf.nn.rnn_cell.BasicLSTMCell(rnn_dim, initializer = init) + self.params['state'] = tf.Variable(tf.zeros([self.batch_size, rnn_dim]), trainable=False) + self.params['output'] = tf.Variable(tf.zeros([self.batch_size, rnn_dim]), trainable=False) + + + def nn_step(self, X, state): + X = tf.convert_to_tensor(X) + return self.params['cell'](X, state) + + def zero_state(self): + return (self.params['output'], self.params['state']) diff --git a/copy/static/dnc_schema.png b/repeat-copy/static/dnc_schema.png similarity index 100% rename from copy/static/dnc_schema.png rename to repeat-copy/static/dnc_schema.png