diff --git a/Notebooks/Chap16/16_2_Autoregressive_Flows.ipynb b/Notebooks/Chap16/16_2_Autoregressive_Flows.ipynb new file mode 100644 index 0000000..7d5839d --- /dev/null +++ b/Notebooks/Chap16/16_2_Autoregressive_Flows.ipynb @@ -0,0 +1,307 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "authorship_tag": "ABX9TyMe8jb5kLJqkNSE/AwExTpa", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# **Notebook 16.2: 1D autoregressive flows**\n", + "\n", + "This notebook investigates a 1D normalizing flows example similar to that illustrated in figure 16.7 in the book.\n", + "\n", + "Work through the cells below, running each cell in turn. In various places you will see the words \"TO DO\". Follow the instructions at these places and make predictions about what is going to happen or write code to complete the functions.\n", + "\n", + "Contact me at udlbookmail@gmail.com if you find any mistakes or have any suggestions." + ], + "metadata": { + "id": "t9vk9Elugvmi" + } + }, + { + "cell_type": "code", + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ], + "metadata": { + "id": "OLComQyvCIJ7" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "First we'll define an invertible one dimensional function as in figure 16.5" + ], + "metadata": { + "id": "jTK456TUd2FV" + } + }, + { + "cell_type": "code", + "source": [ + "# First let's make the 1D piecewise linear mapping as illustated in figure 16.5\n", + "def g(h, phi):\n", + " # TODO -- write this function (equation 16.12)\n", + " # Note: If you have the first printing of the book, there is a mistake in equation 16.12\n", + " # Check the errata for the correct equation (or figure it out yourself!)\n", + " # Replace this line:\n", + " h_prime = 1\n", + "\n", + "\n", + " return h_prime" + ], + "metadata": { + "id": "zceww_9qFi00" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Let's test this out. If you managed to vectorize the routine above, then good for you\n", + "# but I'll assume you didn't and so we'll use a loop\n", + "\n", + "# Define the parameters\n", + "phi = np.array([0.2, 0.1, 0.4, 0.05, 0.25])\n", + "\n", + "# Run the function on an array\n", + "h = np.arange(0,1,0.01)\n", + "h_prime = np.zeros_like(h)\n", + "for i in range(len(h)):\n", + " h_prime[i] = g(h[i], phi)\n", + "\n", + "# Draw the function\n", + "fig, ax = plt.subplots()\n", + "ax.plot(h,h_prime, 'b-')\n", + "ax.set_xlim([0,1])\n", + "ax.set_ylim([0,1])\n", + "ax.set_xlabel('Input, $h$')\n", + "ax.set_ylabel('Output, $h^\\prime$')\n", + "plt.show()\n" + ], + "metadata": { + "id": "CLXhYl9ZIuRN" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "We will also need the inverse of this function" + ], + "metadata": { + "id": "zOCMYC0leOyZ" + } + }, + { + "cell_type": "code", + "source": [ + "# Define the inverse function\n", + "def g_inverse(h_prime, phi):\n", + " # Lot's of ways to do this, but we'll just do it by bracketing\n", + " h_low = 0\n", + " h_mid = 0.5\n", + " h_high = 0.999\n", + "\n", + " thresh = 0.0001\n", + " c_iter = 0\n", + " while(c_iter < 20 and h_high - h_low > thresh):\n", + " h_prime_low = g(h_low, phi)\n", + " h_prime_mid = g(h_mid, phi)\n", + " h_prime_high = g(h_high, phi)\n", + " if h_prime_mid < h_prime:\n", + " h_low = h_mid\n", + " else:\n", + " h_high = h_mid\n", + "\n", + " h_mid = h_low+(h_high-h_low)/2\n", + " c_iter+=1\n", + "\n", + " return h_mid" + ], + "metadata": { + "id": "OIqFAgobeSM8" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Now let's define an autogressive flow. Let's switch to looking at figure 16.7.# We'll assume that our piecewise function will use five parameters phi1,phi2,phi3,phi4,phi5" + ], + "metadata": { + "id": "t8XPxipfd7hz" + } + }, + { + "cell_type": "code", + "source": [ + "\n", + "def ReLU(preactivation):\n", + " activation = preactivation.clip(0.0)\n", + " return activation\n", + "\n", + "def softmax(x):\n", + " x = np.exp(x) ;\n", + " x = x/ np.sum(x) ;\n", + " return x\n", + "\n", + "# Return value of phi that doesn't depend on any of the iputs\n", + "def get_phi():\n", + " return np.array([0.2, 0.1, 0.4, 0.05, 0.25])\n", + "\n", + "# Compute values of phi that depend on h1\n", + "def shallow_network_phi_h1(h1, n_hidden=10):\n", + " # For neatness of code, we'll just define the parameters in the network definition itself\n", + " n_input = 1\n", + " np.random.seed(n_input)\n", + " beta0 = np.random.normal(size=(n_hidden,1))\n", + " Omega0 = np.random.normal(size=(n_hidden, n_input))\n", + " beta1 = np.random.normal(size=(5,1))\n", + " Omega1 = np.random.normal(size=(5, n_hidden))\n", + " return softmax(beta1 + Omega1 @ ReLU(beta0 + Omega0 @ np.array([[h1]])))\n", + "\n", + "# Compute values of phi that depend on h1 and h2\n", + "def shallow_network_phi_h1h2(h1,h2,n_hidden=10):\n", + " # For neatness of code, we'll just define the parameters in the network definition itself\n", + " n_input = 2\n", + " np.random.seed(n_input)\n", + " beta0 = np.random.normal(size=(n_hidden,1))\n", + " Omega0 = np.random.normal(size=(n_hidden, n_input))\n", + " beta1 = np.random.normal(size=(5,1))\n", + " Omega1 = np.random.normal(size=(5, n_hidden))\n", + " return softmax(beta1 + Omega1 @ ReLU(beta0 + Omega0 @ np.array([[h1],[h2]])))\n", + "\n", + "# Compute values of phi that depend on h1, h2, and h3\n", + "def shallow_network_phi_h1h2h3(h1,h2,h3, n_hidden=10):\n", + " # For neatness of code, we'll just define the parameters in the network definition itself\n", + " n_input = 3\n", + " np.random.seed(n_input)\n", + " beta0 = np.random.normal(size=(n_hidden,1))\n", + " Omega0 = np.random.normal(size=(n_hidden, n_input))\n", + " beta1 = np.random.normal(size=(5,1))\n", + " Omega1 = np.random.normal(size=(5, n_hidden))\n", + " return softmax(beta1 + Omega1 @ ReLU(beta0 + Omega0 @ np.array([[h1],[h2],[h3]])))" + ], + "metadata": { + "id": "PnHGlZtcNEAI" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The forward mapping as shown in figure 16.7 a" + ], + "metadata": { + "id": "8fXeG4V44GVH" + } + }, + { + "cell_type": "code", + "source": [ + "def forward_mapping(h1,h2,h3,h4):\n", + " #TODO implement the forward mapping\n", + " #Replace this line:\n", + " h_prime1 = 0 ; h_prime2=0; h_prime3=0; h_prime4 = 0\n", + "\n", + " return h_prime1, h_prime2, h_prime3, h_prime4" + ], + "metadata": { + "id": "N1zjnIoX0TRP" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "The backward mapping as shown in figure 16.7b" + ], + "metadata": { + "id": "H8vQfFwI4L7r" + } + }, + { + "cell_type": "code", + "source": [ + "def backward_mapping(h1_prime,h2_prime,h3_prime,h4_prime):\n", + " #TODO implement the backward mapping\n", + " #Replace this line:\n", + " h1=0; h2=0; h3=0; h4 = 0\n", + "\n", + " return h1,h2,h3,h4" + ], + "metadata": { + "id": "HNcQTiVE4DMJ" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "Finally, let's make sure that the network really can be inverted" + ], + "metadata": { + "id": "W2IxFkuyZJyn" + } + }, + { + "cell_type": "code", + "source": [ + "# Test the network to see if it does invert correctly\n", + "h1 = 0.22; h2 = 0.41; h3 = 0.83; h4 = 0.53\n", + "print(\"Original h values %3.3f,%3.3f,%3.3f,%3.3f\"%(h1,h2,h3,h4))\n", + "h1_prime, h2_prime, h3_prime, h4_prime = forward_mapping(h1,h2,h3,h4)\n", + "print(\"h_prime values %3.3f,%3.3f,%3.3f,%3.3f\"%(h1_prime,h2_prime,h3_prime,h4_prime))\n", + "h1,h2,h3,h4 = backward_mapping(h1_prime,h2_prime,h3_prime,h4_prime)\n", + "print(\"Reconstructed h values %3.3f,%3.3f,%3.3f,%3.3f\"%(h1,h2,h3,h4))" + ], + "metadata": { + "id": "RT7qvEFp700I" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "sDknSPMLZmzh" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file