Created using Colaboratory
This commit is contained in:
udlbook
362
Notebooks/Chap04/4_1_Composing_Networks.ipynb
Normal file
362
Notebooks/Chap04/4_1_Composing_Networks.ipynb
Normal file
@@ -0,0 +1,362 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": [],
|
||||
"authorship_tag": "ABX9TyPEQEGetZqWnLRNn99Q2aaT",
|
||||
"include_colab_link": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "view-in-github",
|
||||
"colab_type": "text"
|
||||
},
|
||||
"source": [
|
||||
"<a href=\"https://colab.research.google.com/github/udlbook/udlbook/blob/main/Notebooks/Chap04/4_1_Composing_Networks.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"#Notebook 4.1 -- Composing networks\n",
|
||||
"\n",
|
||||
"The purpose of this notebook is to understand what happens when we feed one neural network into another. It works through an example similar to 4.1 and varies both networks\n",
|
||||
"\n",
|
||||
"Work through the cells below, running each cell in turn. In various places you will see the words \"TO DO\". Follow the instructions at these places and make predictions about what is going to happen or write code to complete the functions.\n",
|
||||
"\n",
|
||||
"Contact me at udlbookmail@gmail.com if you find any mistakes or have any suggestions"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "MaKn8CFlzN8E"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8ClURpZQzI6L"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Imports math library\n",
|
||||
"import numpy as np\n",
|
||||
"# Imports plotting library\n",
|
||||
"import matplotlib.pyplot as plt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Define the Rectified Linear Unit (ReLU) function\n",
|
||||
"def ReLU(preactivation):\n",
|
||||
" activation = preactivation.clip(0.0)\n",
|
||||
" return activation"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "YdmveeAUz4YG"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Define a shallow neural network with, one input, one output, and three hidden units\n",
|
||||
"def shallow_1_1_3(x, activation_fn, phi_0,phi_1,phi_2,phi_3, theta_10, theta_11, theta_20, theta_21, theta_30, theta_31):\n",
|
||||
" # Initial lines\n",
|
||||
" pre_1 = theta_10 + theta_11 * x\n",
|
||||
" pre_2 = theta_20 + theta_21 * x\n",
|
||||
" pre_3 = theta_30 + theta_31 * x\n",
|
||||
" # Activation functions\n",
|
||||
" act_1 = activation_fn(pre_1)\n",
|
||||
" act_2 = activation_fn(pre_2)\n",
|
||||
" act_3 = activation_fn(pre_3)\n",
|
||||
" # Weight activations\n",
|
||||
" w_act_1 = phi_1 * act_1\n",
|
||||
" w_act_2 = phi_2 * act_2\n",
|
||||
" w_act_3 = phi_3 * act_3\n",
|
||||
" # Combine weighted activation and add y offset\n",
|
||||
" y = phi_0 + w_act_1 + w_act_2 + w_act_3\n",
|
||||
" # Return everything we have calculated\n",
|
||||
" return y"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ximCLwIfz8kj"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# # Plot two shallow neural networks and the composition of the two\n",
|
||||
"def plot_neural_two_components(x_in, net1_out, net2_out, net12_out=None):\n",
|
||||
"\n",
|
||||
" # Plot the two networks separately\n",
|
||||
" fig, ax = plt.subplots(1,2)\n",
|
||||
" fig.set_size_inches(8.5, 8.5)\n",
|
||||
" fig.tight_layout(pad=3.0)\n",
|
||||
" ax[0].plot(x_in, net1_out,'r-')\n",
|
||||
" ax[0].set_xlabel('Net 1 input'); ax[0].set_ylabel('Net 1 output')\n",
|
||||
" ax[0].set_xlim([-1,1]);ax[0].set_ylim([-1,1])\n",
|
||||
" ax[0].set_aspect(1.0)\n",
|
||||
" ax[1].plot(x_in, net2_out,'b-')\n",
|
||||
" ax[1].set_xlabel('Net 2 input'); ax[1].set_ylabel('Net 2 output')\n",
|
||||
" ax[1].set_xlim([-1,1]);ax[1].set_ylim([-1,1])\n",
|
||||
" ax[1].set_aspect(1.0)\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
" if net12_out is not None:\n",
|
||||
" # Plot their composition\n",
|
||||
" fig, ax = plt.subplots()\n",
|
||||
" ax.plot(x_in ,net12_out,'g-')\n",
|
||||
" ax.set_xlabel('Net 1 Input'); ax.set_ylabel('Net 2 Output')\n",
|
||||
" ax.set_xlim([-1,1]);ax.set_ylim([-1,1])\n",
|
||||
" ax.set_aspect(1.0)\n",
|
||||
" plt.show()"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ZB2HTalOE40X"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Let's define two networks. We'll put the prefixes n1_ and n2_ before all the variables to make it clear which network is which. We'll just consider the inputs and outputs over the range [-1,1]. If you set the \"plot_all\" flat to True, you can see the details of how they were created."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "LxBJCObC-NTY"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Now lets define some parameters and run the first neural network\n",
|
||||
"n1_theta_10 = 0.0 ; n1_theta_11 = -1.0\n",
|
||||
"n1_theta_20 = 0 ; n1_theta_21 = 1.0\n",
|
||||
"n1_theta_30 = -0.67 ; n1_theta_31 = 1.0\n",
|
||||
"n1_phi_0 = 1.0; n1_phi_1 = -2.0; n1_phi_2 = -3.0; n1_phi_3 = 9.3\n",
|
||||
"\n",
|
||||
"# Now lets define some parameters and run the second neural network\n",
|
||||
"n2_theta_10 = -0.6 ; n2_theta_11 = -1.0\n",
|
||||
"n2_theta_20 = 0.2 ; n2_theta_21 = 1.0\n",
|
||||
"n2_theta_30 = -0.5 ; n2_theta_31 = 1.0\n",
|
||||
"n2_phi_0 = 0.5; n2_phi_1 = -1.0; n2_phi_2 = -1.5; n2_phi_3 = 2.0\n",
|
||||
"\n",
|
||||
"# Display the two inputs\n",
|
||||
"x = np.arange(-1,1,0.001)\n",
|
||||
"# We run the first and second neural networks for each of these input values\n",
|
||||
"net1_out = shallow_1_1_3(x, ReLU, n1_phi_0, n1_phi_1, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"net2_out = shallow_1_1_3(x, ReLU, n2_phi_0, n2_phi_1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"# Plot both graphs\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "JRebvurv22pT"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"# Take a piece of paper and draw what you think will happen when we feed the\n",
|
||||
"# output of the first network into the second one. Draw the relationship between\n",
|
||||
"# the input of the first network and the output of the second one."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "NUQVop9-Xta1"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Now let's see if your predictions were right\n",
|
||||
"\n",
|
||||
"# TODO feed the output of first network into second network (replace this line)\n",
|
||||
"net12_out = np.zeros_like(x)\n",
|
||||
"\n",
|
||||
"# Plot all three graphs\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out, net12_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "Yq7GH-MCIyPI"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Now we'll change things a up a bit. What happens if we change the second network? (note the *-1 change)\n",
|
||||
"net1_out = shallow_1_1_3(x, ReLU, n1_phi_0, n1_phi_1, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"net2_out = shallow_1_1_3(x, ReLU, n2_phi_0, n2_phi_1*-1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "BMlLkLbdEuPu"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"# Take a piece of paper and draw what you think will happen when we feed the\n",
|
||||
"# output of the first network into the second one now that we have changed it. Draw the relationship between\n",
|
||||
"# the input of the first network and the output of the second one."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "Of6jVXLTJ688"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# When you have a prediction, run this code to see if you were right\n",
|
||||
"net12_out = shallow_1_1_3(net1_out, ReLU, n2_phi_0, n2_phi_1*-1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out, net12_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "PbbSCaSeK6SM"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Let's change things again. What happens if we change the first network? (note the changes)\n",
|
||||
"net1_out = shallow_1_1_3(x, ReLU, n1_phi_0, n1_phi_1*0.5, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"net2_out = shallow_1_1_3(x, ReLU, n2_phi_0, n2_phi_1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "b39mcSGFK9Fd"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"# Take a piece of paper and draw what you think will happen when we feed the\n",
|
||||
"# output of the first network now we have changed it into the original second network. Draw the relationship between\n",
|
||||
"# the input of the first network and the output of the second one."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "MhO40cC_LW9I"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# When you have a prediction, run this code to see if you were right\n",
|
||||
"net12_out = shallow_1_1_3(net1_out, ReLU, n2_phi_0, n2_phi_1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out, net12_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "Akwo-hnPLkNr"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Let's change things again. What happens if the first network and second networks are the same?\n",
|
||||
"net1_out = shallow_1_1_3(x, ReLU, n1_phi_0, n1_phi_1, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"net2_out_new = shallow_1_1_3(x, ReLU, n1_phi_0, n1_phi_1, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out_new)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "TJ7wXKpRLl_E"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"# Take a piece of paper and draw what you think will happen when we feed the\n",
|
||||
"# output of the first network into the original second network. Draw the relationship between\n",
|
||||
"# the input of the first network and the output of the second one."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "dJbbh6R7NG9k"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# When you have a prediction, run this code to see if you were right\n",
|
||||
"net12_out = shallow_1_1_3(net1_out, ReLU, n1_phi_0, n1_phi_1, n1_phi_2, n1_phi_3, n1_theta_10, n1_theta_11, n1_theta_20, n1_theta_21, n1_theta_30, n1_theta_31)\n",
|
||||
"plot_neural_two_components(x, net1_out, net2_out_new, net12_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "BiZZl3yNM2Bq"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TODO\n",
|
||||
"# Contemplate what you think will happen when we feed the\n",
|
||||
"# output of the original first network into a second copy of the original first network, and then\n",
|
||||
"# the output of that into the original second network (so now we have a three layer network)\n",
|
||||
"# How many total linear regions will we have in the output?\n",
|
||||
"net123_out = shallow_1_1_3(net12_out, ReLU, n2_phi_0, n2_phi_1, n2_phi_2, n2_phi_3, n2_theta_10, n2_theta_11, n2_theta_20, n2_theta_21, n2_theta_30, n2_theta_31)\n",
|
||||
"plot_neural_two_components(x, net12_out, net2_out, net123_out)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "BSd51AkzNf7-"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# TO DO\n",
|
||||
"# How many linear regions would there be if we ran N copies of the first network, feeding the result of the first\n",
|
||||
"# into the second, the second into the third and so on, and then passed the result into the original second\n",
|
||||
"# network (blue curve above)\n",
|
||||
"\n",
|
||||
"# Take away conclusion: with very few parameters, we can make A LOT of linear regions, but\n",
|
||||
"# they depend on one another in complex ways that quickly become to difficult to understand intuitively."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "HqzePCLOVQK7"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user