{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c0",
   "metadata": {},
   "source": [
    "# Intro to deep learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from platform import python_version\n",
    "print(python_version())\n",
    "import torch\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import json\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import scipy.stats as st\n",
    "import scipy.integrate as integrate\n",
    "from scipy.stats import multivariate_normal\n",
    "from sklearn import linear_model\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier, MLPRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from scipy.special import erf\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "from matplotlib.colors import LogNorm\n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_palette(\"colorblind\")\n",
    "palette = sns.color_palette()\n",
    "figsize = (15,8)\n",
    "legend_fontsize = 16\n",
    "\n",
    "from matplotlib import rc\n",
    "rc('font',**{'family':'sans-serif'})\n",
    "rc('text', usetex=False)\n",
    "rc('text.latex',preamble=r'\\usepackage[utf8]{inputenc}')\n",
    "rc('text.latex',preamble=r'\\usepackage[english]{babel}')\n",
    "rc('figure', **{'dpi': 300})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3",
   "metadata": {},
   "source": [
    "## A single neuron cannot solve XOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[0,0],[0,1],[1,0],[1,1]], float); yv = np.array([0,1,1,0])\n",
    "gx, gy = np.meshgrid(np.linspace(-0.3,1.3,300), np.linspace(-0.3,1.3,300))\n",
    "grid = np.c_[gx.ravel(), gy.ravel()]\n",
    "fig, axes = plt.subplots(1, 2, figsize=(13, 6))\n",
    "for ax,(title,m) in zip(axes, [(\"Single neuron (logistic regression)\", LogisticRegression()),\n",
    "        (\"MLP, one hidden layer of 8\", make_pipeline(StandardScaler(),\n",
    "            MLPClassifier(hidden_layer_sizes=(8,), activation=\"tanh\", solver=\"lbfgs\", max_iter=5000, random_state=0)))]):\n",
    "    m.fit(X, yv)\n",
    "    Z = m.predict_proba(grid)[:, 1].reshape(gx.shape)\n",
    "    cf = ax.contourf(gx, gy, Z, levels=np.linspace(0,1,21), cmap=\"RdBu_r\", vmin=0, vmax=1)\n",
    "    ax.contour(gx, gy, Z, levels=[0.5], colors=\"k\", linewidths=2)\n",
    "    ax.scatter(X[:,0], X[:,1], c=yv, cmap=\"RdBu_r\", vmin=0, vmax=1, edgecolors=\"k\", s=400, linewidths=2, zorder=3)\n",
    "    ax.set_title(title, fontsize=legend_fontsize); ax.set_aspect(\"equal\")\n",
    "fig.colorbar(cf, ax=axes, shrink=0.8, label=\"P(class 1)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5",
   "metadata": {},
   "source": [
    "## Universal approximation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = lambda x: np.sin(2*x) + 0.5*np.cos(5*x)\n",
    "Xtr = np.linspace(-3, 3, 80)[:, None]; ytr = f(Xtr).ravel()\n",
    "xs = np.linspace(-3, 3, 400)[:, None]\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "ax.plot(xs, f(xs), color=\"black\", linewidth=2, label=\"Target function\")\n",
    "ax.scatter(Xtr, ytr, marker=\"*\", s=50, color=palette[7], zorder=3)\n",
    "for h in [2, 5, 50]:\n",
    "    m = make_pipeline(StandardScaler(),\n",
    "                      MLPRegressor(hidden_layer_sizes=(h,), activation=\"tanh\", solver=\"lbfgs\",\n",
    "                                   max_iter=5000, random_state=0))\n",
    "    m.fit(Xtr, ytr)\n",
    "    ax.plot(xs, m.predict(xs), linewidth=2, label=\"%d hidden units\" % h)\n",
    "ax.set_ylim((-2, 2)); ax.legend(loc=\"upper right\", fontsize=legend_fontsize)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "thresh = np.vectorize(lambda x : 1 if x >= 0 else 0)\n",
    "\n",
    "ax.plot(xs, thresh(xs), linewidth=lw, label=\"Threshold activation\")\n",
    "ax.plot(xs, 1. / (1 + np.exp(-xs)), linewidth=lw, label=\"Logistic sigmoid\")\n",
    "ax.plot(xs, np.tanh(xs), linewidth=lw, label=\"Hyperbolic tangent $\\\\tanh$\")\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "\n",
    "ax.set_ylim((-1., 2.))\n",
    "ax.set_xlim((-5., 5.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "softplus = np.vectorize(lambda x : np.log(1 + np.exp(x)))\n",
    "lrelu = np.vectorize(lambda x : x if x >= 0 else 0.2*x)\n",
    "lrelu2 = np.vectorize(lambda x : x if x >= 0 else 0.05*x)\n",
    "elu = np.vectorize(lambda x : x if x >= 0 else 1.0*(np.exp(x)-1))\n",
    "elu2 = np.vectorize(lambda x : x if x >= 0 else 0.2*(np.exp(x)-1))\n",
    "thresh = np.vectorize(lambda x : 1 if x >= 0 else 0)\n",
    "\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "ax.plot(xs, softplus(xs), linewidth=lw, label=\"Softplus\")\n",
    "ax.plot(xs, lrelu(xs), linewidth=lw, label=\"Leaky ReLU, $a=\\\\frac{1}{5}$\")\n",
    "ax.plot(xs, lrelu2(xs), linewidth=lw, label=\"Leaky ReLU, $a=\\\\frac{1}{20}$\")\n",
    "ax.plot(xs, elu(xs), linewidth=lw, label=\"Exponential linear unit, $\\\\alpha=1.0$\")\n",
    "ax.plot(xs, elu2(xs), linewidth=lw, label=\"Exponential linear unit, $\\\\alpha=\\\\frac{1}{5}$\")\n",
    "\n",
    "ax.set_ylim((-1., 1.))\n",
    "ax.set_xlim((-5., 1.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "swish = np.vectorize( lambda x : x / (1. + np.exp(-x)))\n",
    "swish2 = np.vectorize( lambda x : x / (1. + np.exp(-5*x)))\n",
    "mish = np.vectorize(lambda x : x * np.tanh(np.log(1 + np.exp(x))))\n",
    "\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "ax.plot(xs, swish(xs), linewidth=lw, label=\"Swish activation, $\\\\beta=1$\")\n",
    "ax.plot(xs, swish2(xs), linewidth=lw, label=\"Swish activation, $\\\\beta=5$\")\n",
    "ax.plot(xs, mish(xs), linewidth=lw, label=\"Mish activation\")\n",
    "\n",
    "ax.set_ylim((-.5, 2.))\n",
    "ax.set_xlim((-4., 2.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c10",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "def aconc(a1, a2, beta):\n",
    "    return lambda x : (a1-a2)*x / (1.0 + np.exp(-x*beta*(a1-a2))) + a2*x\n",
    "\n",
    "acon1 = np.vectorize( aconc(1.0, 0.0, 1.0) )\n",
    "acon2 = np.vectorize( aconc(1.2, -0.1, 1.0) )\n",
    "acon3 = np.vectorize( aconc(1.0, -0.8, 1.0) )\n",
    "acon4 = np.vectorize( aconc(1.0, -0.8, 0.1) )\n",
    "acon5 = np.vectorize( aconc(1.0, -0.8, 0.01) )\n",
    "\n",
    "ax.plot(xs, acon1(xs), linewidth=lw, label=\"ACON-C, $a_1=1$, $a_2=0$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon2(xs), linewidth=lw, label=\"ACON-C, $a_1=1.2$, $a_2=-0.1$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon3(xs), linewidth=lw, color=\"C3\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon4(xs), linewidth=lw, color=\"C3\", linestyle=\"dashed\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=0.1$\")\n",
    "ax.plot(xs, acon5(xs), linewidth=lw, color=\"C3\", linestyle=\"dotted\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=0.01$\")\n",
    "\n",
    "ax.set_ylim((-.5, 4.))\n",
    "ax.set_xlim((-4., 4.))\n",
    "ax.legend(loc=\"upper center\")\n",
    "# plt.savefig('act4.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c11",
   "metadata": {},
   "source": [
    "## Activation functions and their derivatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c12",
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = np.linspace(-5, 5, 500)\n",
    "sig = 1/(1+np.exp(-xs)); tnh = np.tanh(xs); relu = np.maximum(0, xs)\n",
    "gelu = xs*0.5*(1+erf(xs/np.sqrt(2)))\n",
    "fig, axes = plt.subplots(2, 1, figsize=(12, 10))\n",
    "for v, l in [(sig, r\"sigmoid\"), (tnh, r\"$\\tanh$\"), (relu, r\"ReLU\"), (gelu, r\"GELU\")]:\n",
    "    axes[0].plot(xs, v, linewidth=2, label=l)\n",
    "for v, l in [(sig*(1-sig), r\"sigmoid\"), (1-tnh**2, r\"$\\tanh$\"), ((xs > 0).astype(float), r\"ReLU\"),\n",
    "             (0.5*(1+erf(xs/np.sqrt(2)))+xs*np.exp(-xs**2/2)/np.sqrt(2*np.pi), r\"GELU\")]:\n",
    "    axes[1].plot(xs, v, linewidth=2, label=l)\n",
    "axes[0].set_title(r\"Activations\"); axes[0].set_ylim((-1.2, 2)); axes[0].legend(loc=\"upper left\")\n",
    "axes[1].set_title(r\"Derivatives\"); axes[1].set_ylim((-0.2, 1.2)); axes[1].legend(loc=\"upper left\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c13",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([1.0], requires_grad=True)\n",
    "y = torch.tensor([1.0], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c14",
   "metadata": {},
   "outputs": [],
   "source": [
    "rho = 1.0\n",
    "f = x * x + rho * y * y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c15",
   "metadata": {},
   "outputs": [],
   "source": [
    "f.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c16",
   "metadata": {},
   "source": [
    "## Backpropagation on a computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c17",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "x0, y0 = 1.0, 2.0\n",
    "fwd = {\"x\": x0, \"y\": y0, \"sqx\": x0**2, \"mul\": x0*y0, \"add\": x0+y0}\n",
    "fwd[\"sqc\"], fwd[\"add2\"] = fwd[\"add\"]**2, fwd[\"sqx\"]+fwd[\"mul\"]; fwd[\"f\"] = fwd[\"add2\"] + fwd[\"sqc\"]\n",
    "bwd = {\"f\": 1.0}; bwd[\"add2\"] = bwd[\"f\"]; bwd[\"sqc\"] = bwd[\"f\"]\n",
    "bwd[\"sqx\"] = bwd[\"add2\"]; bwd[\"mul\"] = bwd[\"add2\"]; bwd[\"add\"] = bwd[\"sqc\"] * 2*fwd[\"add\"]\n",
    "bwd[\"x\"] = bwd[\"sqx\"]*2*x0 + bwd[\"mul\"]*y0 + bwd[\"add\"]; bwd[\"y\"] = bwd[\"mul\"]*x0 + bwd[\"add\"]\n",
    "labels = {\"x\":\"$x$\",\"y\":\"$y$\",\"sqx\":\"$(\\\\cdot)^2$\",\"mul\":\"$\\\\times$\",\"add\":\"$+$\",\"sqc\":\"$(\\\\cdot)^2$\",\"add2\":\"$+$\",\"f\":\"$+$\"}\n",
    "pos = {\"x\":(0,2.7),\"y\":(0,0.3),\"sqx\":(1.6,3.5),\"mul\":(1.6,2.0),\"add\":(1.6,0.3),\"sqc\":(3.2,0.3),\"add2\":(3.2,2.7),\"f\":(4.8,1.6)}\n",
    "edges = [(\"x\",\"sqx\"),(\"x\",\"mul\"),(\"x\",\"add\"),(\"y\",\"mul\"),(\"y\",\"add\"),(\"sqx\",\"add2\"),(\"mul\",\"add2\"),(\"add\",\"sqc\"),(\"add2\",\"f\"),(\"sqc\",\"f\")]\n",
    "G = nx.DiGraph(); G.add_nodes_from(pos); G.add_edges_from(edges)\n",
    "def draw(ax, show_bwd, title):\n",
    "    nx.draw_networkx_edges(G, pos, ax=ax, arrowstyle=\"-|>\", arrowsize=16, edge_color=\"0.55\",\n",
    "                           width=1.5, node_size=2400, min_source_margin=16, min_target_margin=16)\n",
    "    nx.draw_networkx_nodes(G, pos, nodelist=[\"x\",\"y\"], ax=ax, node_size=2400, node_color=\"#fff3d6\", edgecolors=\"black\", linewidths=1.5)\n",
    "    nx.draw_networkx_nodes(G, pos, nodelist=[n for n in pos if n not in (\"x\",\"y\")], ax=ax, node_size=2400, node_color=\"white\", edgecolors=\"black\", linewidths=1.5)\n",
    "    nx.draw_networkx_labels(G, pos, labels, ax=ax, font_size=19)\n",
    "    for n,(px,py) in pos.items():\n",
    "        ax.text(px+0.30, py+0.22, \"%g\" % fwd[n], color=\"green\", fontsize=15, ha=\"left\", va=\"bottom\", weight=\"bold\")\n",
    "        if show_bwd:\n",
    "            ax.text(px+0.30, py-0.22, \"%g\" % bwd[n], color=\"red\", fontsize=15, ha=\"left\", va=\"top\", weight=\"bold\")\n",
    "    ax.set_title(title, fontsize=15); ax.axis(\"off\"); ax.set_xlim(-0.5, 5.7); ax.set_ylim(-0.5, 4.2)\n",
    "fig, axes = plt.subplots(2, 1, figsize=(11, 12))\n",
    "draw(axes[0], False, r\"Forward pass: $f(x,y)=x^2+xy+(x+y)^2$ at $x=1,y=2$   (green = values)\")\n",
    "draw(axes[1], True,  r\"Backward pass   (red = $\\partial f/\\partial\\,\\cdot$;  $\\partial f/\\partial x=10,\\ \\partial f/\\partial y=7$)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c18",
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_func(x, y, a=1, b=10):\n",
    "    return (1.5 - x + x*y)**2 + (2.25 - x + x*y**2)**2 + (2.625 - x + x*y**3)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_results(f, Optimizer, momentum=0, nesterov=False, n=100, lrs=[0.01, 0.001, 0.0001], x0=-2.0, y0=-2.0):\n",
    "    results = {}\n",
    "    for lr in lrs:\n",
    "        x = torch.tensor([x0], requires_grad=True)\n",
    "        y = torch.tensor([y0], requires_grad=True)\n",
    "        x_hist, y_hist = [x0], [y0]\n",
    "        if Optimizer == optim.SGD:\n",
    "            optimizer = Optimizer([x, y], lr=lr, momentum=momentum, nesterov=nesterov)\n",
    "        else:\n",
    "            optimizer = Optimizer([x, y], lr=lr)\n",
    "        def closure():\n",
    "            optimizer.zero_grad()\n",
    "            ff = f(x, y)\n",
    "            ff.backward()\n",
    "            return ff\n",
    "        for _ in range(n):\n",
    "            optimizer.step(closure)\n",
    "            x_hist.append(x.detach().numpy()[0])\n",
    "            y_hist.append(y.detach().numpy()[0])\n",
    "        results[lr] = (x_hist, y_hist)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c20",
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_plot(ax, my_func, results, x0=-2.0, y0=-2.0, xopt=1, yopt=1, delta = 0.02, my_limx = 2.5, my_limy = 2.5, legend=True, legendloc=\"lower left\"):\n",
    "    xs = np.arange(-my_limx, my_limx, delta)\n",
    "    ys = np.arange(-my_limy, my_limy, delta)\n",
    "    Xs, Ys = np.meshgrid(xs, ys)\n",
    "    Zs = my_func(Xs, Ys)\n",
    "    CS = ax.contour(Xs, Ys, Zs, levels=np.logspace(0, 5, 100), norm=LogNorm(), cmap=plt.cm.jet, linewidths=.1)\n",
    "    ax.scatter([x0], [y0], marker='*', s=40, color='g')\n",
    "    ax.scatter([xopt], [yopt], marker='*', s=40, color='r')\n",
    "    for label, hist in results:\n",
    "        ax.plot(hist[0], hist[1], linewidth='1.0', label=label)\n",
    "        ax.scatter(hist[0][-1], hist[1][-1], marker='*', s=20)\n",
    "    ax.set_xlim((-my_limx, my_limx))\n",
    "    ax.set_ylim((-my_limy, my_limy))\n",
    "    if legend:\n",
    "        ax.legend(framealpha=1, loc=legendloc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0, rho = 1., 1.5, .001\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + rho * (y**2)\n",
    "\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['GD, $n=1000$'] = compute_results(my_func2, optim.SGD, x0=x0, y0=y0, lrs=[0.5,.9, .99], n=2000)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "# to_plot = [ (\"SGD, $n=100$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=100$'].items() ] + [ (\"SGD, $n=1000$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "to_plot = [ (\"GD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['GD, $n=1000$'].items() ]\n",
    "my_plot(ax, my_func2, to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "plt.ylim((-.5, 2.))\n",
    "# plt.savefig('sgd0.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c22",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + 0.01 * (y**2)\n",
    "\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['sgd'] = compute_results(my_func2, optim.SGD, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "results['mom'] = compute_results(my_func2, optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "results['nag'] = compute_results(my_func2, optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ]  + \\\n",
    "  [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "my_plot(ax, my_func2, to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "plt.ylim((-1.5, 1.75))\n",
    "plt.xlim((-.5, .5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c23",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "\n",
    "def my_func1(x, y):\n",
    "    return x**2 + 1. * (y**2)\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + 0.1 * (y**2)\n",
    "\n",
    "def my_func3(x, y):\n",
    "    return x**2 + 0.01 * (y**2)\n",
    "\n",
    "fig, axs = plt.subplots(1, 3, figsize=(8,5), sharey=True)\n",
    "my_funcs = [my_func1, my_func2, my_func3]\n",
    "\n",
    "for i in range(3):\n",
    "    results = {}\n",
    "    results['sgd'] = compute_results(my_funcs[i], optim.SGD, x0=x0, y0=y0, lrs=[0.05, 0.01], n=500)\n",
    "#     results['mom'] = compute_results(my_funcs[i], optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.05], n=200)\n",
    "#     results['nag'] = compute_results(my_funcs[i], optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.05], n=200)\n",
    "\n",
    "    to_plot = [ (\"SGD, $\\\\alpha=%s$\" % lr, hist) for lr, hist in results['sgd'].items() ] \n",
    "#       [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ] + \\\n",
    "#       [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "\n",
    "    my_plot(axs[i], my_funcs[i], to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "\n",
    "plt.ylim((-1., 2.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c24",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['SGD, $n=1000$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.05, 0.01, 0.005, 0.001], n=500)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "# to_plot = [ (\"SGD, $n=100$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=100$'].items() ] + [ (\"SGD, $n=1000$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=500$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3)\n",
    "plt.ylim((-.5, 2.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c25",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "results = {}\n",
    "results['sgd'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "results['mom'] = compute_results(my_func, optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "results['nag'] = compute_results(my_func, optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ] + \\\n",
    "  [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3, legendloc=\"upper right\")\n",
    "plt.ylim((-2., 2.))\n",
    "plt.xlim((0., 4.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c26",
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "n = 500\n",
    "results = {}\n",
    "results['sgd'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001], n=n)\n",
    "results['adagrad'] = compute_results(my_func, optim.Adagrad, x0=x0, y0=y0, lrs=[1.], n=n)\n",
    "results['adadelta'] = compute_results(my_func, optim.Adadelta, x0=x0, y0=y0, lrs=[1.], n=n)\n",
    "results['adam'] = compute_results(my_func, optim.Adam, x0=x0, y0=y0, lrs=[0.1], n=n)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"Adagrad, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adagrad'].items() ] + \\\n",
    "  [ (\"Adadelta, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adadelta'].items() ] + \\\n",
    "  [ (\"Adam, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adam'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3, legendloc=\"upper right\")\n",
    "plt.ylim((-.5, 1.6))\n",
    "plt.xlim((0, 3.5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}