{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "# Как обучать нейронные сети\n\nИТМО, лекция 2, 21 сентября 2026 г."
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from platform import python_version\n",
    "print(python_version()\n",
    "     )\n",
    "import torch\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import json\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import scipy.stats as st\n",
    "import scipy.integrate as integrate\n",
    "from scipy.stats import multivariate_normal\n",
    "from sklearn import linear_model\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "from matplotlib.colors import LogNorm\n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_palette(\"colorblind\")\n",
    "palette = sns.color_palette()\n",
    "figsize = (15,8)\n",
    "legend_fontsize = 16\n",
    "\n",
    "from matplotlib import rc\n",
    "rc('font',**{'family':'sans-serif'})\n",
    "rc('text', usetex=True)\n",
    "rc('text.latex',preamble=r'\\usepackage[utf8]{inputenc}')\n",
    "rc('text.latex',preamble=r'\\usepackage[russian]{babel}')\n",
    "# rc('text.latex',preamble=r'\\DeclareUnicodeCharacter{202F}{\\,}')\n",
    "rc('figure', **{'dpi': 300})\n",
    "rc('font',**{'sans-serif': 'Roboto', 'family': 'sans-serif'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Функции активации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "thresh = np.vectorize(lambda x : 1 if x >= 0 else 0)\n",
    "\n",
    "ax.plot(xs, thresh(xs), linewidth=lw, label=\"Threshold activation\")\n",
    "ax.plot(xs, 1. / (1 + np.exp(-xs)), linewidth=lw, label=\"Logistic sigmoid\")\n",
    "ax.plot(xs, np.tanh(xs), linewidth=lw, label=\"Hyperbolic tangent $\\\\tanh$\")\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "\n",
    "ax.set_ylim((-1., 2.))\n",
    "ax.set_xlim((-5., 5.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "softplus = np.vectorize(lambda x : np.log(1 + np.exp(x)))\n",
    "lrelu = np.vectorize(lambda x : x if x >= 0 else 0.2*x)\n",
    "lrelu2 = np.vectorize(lambda x : x if x >= 0 else 0.05*x)\n",
    "elu = np.vectorize(lambda x : x if x >= 0 else 1.0*(np.exp(x)-1))\n",
    "elu2 = np.vectorize(lambda x : x if x >= 0 else 0.2*(np.exp(x)-1))\n",
    "thresh = np.vectorize(lambda x : 1 if x >= 0 else 0)\n",
    "\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "ax.plot(xs, softplus(xs), linewidth=lw, label=\"Softplus\")\n",
    "ax.plot(xs, lrelu(xs), linewidth=lw, label=\"Leaky ReLU, $a=\\\\frac{1}{5}$\")\n",
    "ax.plot(xs, lrelu2(xs), linewidth=lw, label=\"Leaky ReLU, $a=\\\\frac{1}{20}$\")\n",
    "ax.plot(xs, elu(xs), linewidth=lw, label=\"Exponential linear unit, $\\\\alpha=1.0$\")\n",
    "ax.plot(xs, elu2(xs), linewidth=lw, label=\"Exponential linear unit, $\\\\alpha=\\\\frac{1}{5}$\")\n",
    "\n",
    "ax.set_ylim((-1., 1.))\n",
    "ax.set_xlim((-5., 1.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "relu = np.vectorize(lambda x : max(0.0, x))\n",
    "swish = np.vectorize( lambda x : x / (1. + np.exp(-x)))\n",
    "swish2 = np.vectorize( lambda x : x / (1. + np.exp(-5*x)))\n",
    "mish = np.vectorize(lambda x : x * np.tanh(np.log(1 + np.exp(x))))\n",
    "\n",
    "ax.plot(xs, relu(xs), linewidth=lw, label=\"ReLU activation\")\n",
    "ax.plot(xs, swish(xs), linewidth=lw, label=\"Swish activation, $\\\\beta=1$\")\n",
    "ax.plot(xs, swish2(xs), linewidth=lw, label=\"Swish activation, $\\\\beta=5$\")\n",
    "ax.plot(xs, mish(xs), linewidth=lw, label=\"Mish activation\")\n",
    "\n",
    "ax.set_ylim((-.5, 2.))\n",
    "ax.set_xlim((-4., 2.))\n",
    "ax.legend(loc=\"upper left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,5))\n",
    "xs = np.linspace(-5, 5, 500)\n",
    "lw = 1.5\n",
    "\n",
    "def aconc(a1, a2, beta):\n",
    "    return lambda x : (a1-a2)*x / (1.0 + np.exp(-x*beta*(a1-a2))) + a2*x\n",
    "\n",
    "acon1 = np.vectorize( aconc(1.0, 0.0, 1.0) )\n",
    "acon2 = np.vectorize( aconc(1.2, -0.1, 1.0) )\n",
    "acon3 = np.vectorize( aconc(1.0, -0.8, 1.0) )\n",
    "acon4 = np.vectorize( aconc(1.0, -0.8, 0.1) )\n",
    "acon5 = np.vectorize( aconc(1.0, -0.8, 0.01) )\n",
    "\n",
    "ax.plot(xs, acon1(xs), linewidth=lw, label=\"ACON-C, $a_1=1$, $a_2=0$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon2(xs), linewidth=lw, label=\"ACON-C, $a_1=1.2$, $a_2=-0.1$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon3(xs), linewidth=lw, color=\"C3\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=1$\")\n",
    "ax.plot(xs, acon4(xs), linewidth=lw, color=\"C3\", linestyle=\"dashed\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=0.1$\")\n",
    "ax.plot(xs, acon5(xs), linewidth=lw, color=\"C3\", linestyle=\"dotted\", label=\"ACON-C, $a_1=1$, $a_2=-0.8$, $\\\\beta=0.01$\")\n",
    "\n",
    "ax.set_ylim((-.5, 4.))\n",
    "ax.set_xlim((-4., 4.))\n",
    "ax.legend(loc=\"upper center\")\n",
    "# plt.savefig('../figures/act4.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Варианты градиентного спуска"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([1.0], requires_grad=True)\n",
    "y = torch.tensor([1.0], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rho = 1.0\n",
    "f = x * x + rho * y * y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_func(x, y, a=1, b=10):\n",
    "    return (1.5 - x + x*y)**2 + (2.25 - x + x*y**2)**2 + (2.625 - x + x*y**3)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_results(f, Optimizer, momentum=0, nesterov=False, n=100, lrs=[0.01, 0.001, 0.0001], x0=-2.0, y0=-2.0):\n",
    "    results = {}\n",
    "    for lr in lrs:\n",
    "        x = torch.tensor([x0], requires_grad=True)\n",
    "        y = torch.tensor([y0], requires_grad=True)\n",
    "        x_hist, y_hist = [x0], [y0]\n",
    "        if Optimizer == optim.SGD:\n",
    "            optimizer = Optimizer([x, y], lr=lr, momentum=momentum, nesterov=nesterov)\n",
    "        else:\n",
    "            optimizer = Optimizer([x, y], lr=lr)\n",
    "        def closure():\n",
    "            optimizer.zero_grad()\n",
    "            ff = f(x, y)\n",
    "            ff.backward()\n",
    "            return ff\n",
    "        for _ in range(n):\n",
    "            optimizer.step(closure)\n",
    "            x_hist.append(x.detach().numpy()[0])\n",
    "            y_hist.append(y.detach().numpy()[0])\n",
    "        results[lr] = (x_hist, y_hist)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_plot(ax, my_func, results, x0=-2.0, y0=-2.0, xopt=1, yopt=1, delta = 0.02, my_limx = 2.5, my_limy = 2.5, legend=True, legendloc=\"lower left\"):\n",
    "    xs = np.arange(-my_limx, my_limx, delta)\n",
    "    ys = np.arange(-my_limy, my_limy, delta)\n",
    "    Xs, Ys = np.meshgrid(xs, ys)\n",
    "    Zs = my_func(Xs, Ys)\n",
    "    CS = ax.contour(Xs, Ys, Zs, levels=np.logspace(0, 5, 100), norm=LogNorm(), cmap=plt.cm.jet, linewidths=.1)\n",
    "    ax.scatter([x0], [y0], marker='*', s=40, color='g')\n",
    "    ax.scatter([xopt], [yopt], marker='*', s=40, color='r')\n",
    "    for label, hist in results:\n",
    "        ax.plot(hist[0], hist[1], linewidth='1.0', label=label)\n",
    "        ax.scatter(hist[0][-1], hist[1][-1], marker='*', s=20)\n",
    "    ax.set_xlim((-my_limx, my_limx))\n",
    "    ax.set_ylim((-my_limy, my_limy))\n",
    "    if legend:\n",
    "        ax.legend(framealpha=1, loc=legendloc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0, rho = 1., 1.5, .001\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + rho * (y**2)\n",
    "\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['SGD, $n=1000$'] = compute_results(my_func2, optim.SGD, x0=x0, y0=y0, lrs=[.9999, 0.5, .001], n=200)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "# to_plot = [ (\"SGD, $n=100$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=100$'].items() ] + [ (\"SGD, $n=1000$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "my_plot(ax, my_func2, to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "plt.ylim((-.5, 2.))\n",
    "# plt.savefig('../figures/sgd0.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + 0.01 * (y**2)\n",
    "\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['sgd'] = compute_results(my_func2, optim.SGD, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "results['mom'] = compute_results(my_func2, optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "results['nag'] = compute_results(my_func2, optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.1], n=200)\n",
    "\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ]  + \\\n",
    "  [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "my_plot(ax, my_func2, to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "plt.ylim((-1.5, 1.75))\n",
    "plt.xlim((-.5, .5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "\n",
    "def my_func1(x, y):\n",
    "    return x**2 + 1. * (y**2)\n",
    "\n",
    "def my_func2(x, y):\n",
    "    return x**2 + 0.1 * (y**2)\n",
    "\n",
    "def my_func3(x, y):\n",
    "    return x**2 + 0.01 * (y**2)\n",
    "\n",
    "fig, axs = plt.subplots(1, 3, figsize=(8,5), sharey=True)\n",
    "my_funcs = [my_func1, my_func2, my_func3]\n",
    "\n",
    "for i in range(3):\n",
    "    results = {}\n",
    "    results['sgd'] = compute_results(my_funcs[i], optim.SGD, x0=x0, y0=y0, lrs=[0.05, 0.01], n=500)\n",
    "#     results['mom'] = compute_results(my_funcs[i], optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.05], n=200)\n",
    "#     results['nag'] = compute_results(my_funcs[i], optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.05], n=200)\n",
    "\n",
    "    to_plot = [ (\"SGD, $\\\\alpha=%s$\" % lr, hist) for lr, hist in results['sgd'].items() ] \n",
    "#       [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ] + \\\n",
    "#       [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "\n",
    "    my_plot(axs[i], my_funcs[i], to_plot, x0=x0, y0=y0, xopt=0, yopt=0, my_limx=4, my_limy=3)\n",
    "\n",
    "plt.ylim((-1., 2.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "results = {}\n",
    "# results['SGD, $n=100$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001, 0.0001], n=100)\n",
    "results['SGD, $n=1000$'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.05, 0.01, 0.005, 0.001], n=500)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "# to_plot = [ (\"SGD, $n=100$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=100$'].items() ] + [ (\"SGD, $n=1000$, $\\\\alpha=%.4f$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=500$\" % lr, hist) for lr, hist in results['SGD, $n=1000$'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3)\n",
    "plt.ylim((-.5, 2.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "results = {}\n",
    "results['sgd'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "results['mom'] = compute_results(my_func, optim.SGD, momentum=0.95, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "results['nag'] = compute_results(my_func, optim.SGD, momentum=0.95, nesterov=True, x0=x0, y0=y0, lrs=[0.002], n=300)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    " \n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"SGD with momentum, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['mom'].items() ] + \\\n",
    "  [ (\"SGD with NAG, $\\\\gamma=0.95$, $\\\\alpha=%s$, $n=200$\" % lr, hist) for lr, hist in results['nag'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3, legendloc=\"upper right\")\n",
    "plt.ylim((-2., 2.))\n",
    "plt.xlim((0., 4.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x0, y0 = 1., 1.5\n",
    "n = 500\n",
    "results = {}\n",
    "results['sgd'] = compute_results(my_func, optim.SGD, x0=x0, y0=y0, lrs=[0.001], n=n)\n",
    "results['adagrad'] = compute_results(my_func, optim.Adagrad, x0=x0, y0=y0, lrs=[1.], n=n)\n",
    "results['adadelta'] = compute_results(my_func, optim.Adadelta, x0=x0, y0=y0, lrs=[1.], n=n)\n",
    "results['adam'] = compute_results(my_func, optim.Adam, x0=x0, y0=y0, lrs=[0.1], n=n)\n",
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "\n",
    "to_plot = [ (\"SGD, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['sgd'].items() ] + \\\n",
    "  [ (\"Adagrad, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adagrad'].items() ] + \\\n",
    "  [ (\"Adadelta, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adadelta'].items() ] + \\\n",
    "  [ (\"Adam, $\\\\alpha=%s$, $n=%d$\" % (lr, n), hist) for lr, hist in results['adam'].items() ]\n",
    "my_plot(ax, my_func, to_plot, x0=x0, y0=y0, xopt=3, yopt=0.5, my_limx=4, my_limy=3, legendloc=\"upper right\")\n",
    "plt.ylim((-.5, 1.6))\n",
    "plt.xlim((0, 3.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}