{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from collections import OrderedDict\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "\n",
    "from scipy.optimize import fmin_powell\n",
    "from scipy import integrate\n",
    "from scipy import linalg\n",
    "\n",
    "from sklearn import linear_model\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_palette(\"colorblind\")\n",
    "palette = sns.color_palette()\n",
    "figsize = (15,8)\n",
    "legend_fontsize = 16\n",
    "\n",
    "from matplotlib import rc\n",
    "rc('font',**{'family':'sans-serif'})\n",
    "rc('text', usetex=True)\n",
    "rc('text.latex',preamble=r'\\usepackage[utf8]{inputenc}')\n",
    "rc('text.latex',preamble=r'\\usepackage[russian]{babel}')\n",
    "rc('axes', **{'titlesize': '16', 'labelsize': '16'})\n",
    "rc('legend', **{'fontsize': '16'})\n",
    "rc('figure', **{'dpi' : 300})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Модельные методы классификации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_mixture(N, pi, mu1, sigma1, mu2, sigma2):\n",
    "    z = np.array( np.random.rand(N) < pi, dtype=int)\n",
    "    res = np.zeros((N, 2))\n",
    "    res[np.where(z == 1)] = np.random.multivariate_normal(mu1, sigma1, np.sum(z))\n",
    "    res[np.where(z == 0)] = np.random.multivariate_normal(mu2, sigma2, N-np.sum(z))\n",
    "    return z, res\n",
    "\n",
    "def plot_points(ax, x, z, mu1, mu2):\n",
    "    ax.scatter(x[np.where(z==1), 0], x[np.where(z==1), 1], marker='.', color='b')\n",
    "    ax.scatter(x[np.where(z==0), 0], x[np.where(z==0), 1], marker='.', color='r')\n",
    "    ax.scatter([mu1[0]], [mu1[1]], marker='*', s=120, color='b')\n",
    "    ax.scatter([mu2[0]], [mu2[1]], marker='*', s=120, color='r')\n",
    "\n",
    "\n",
    "mu1, sigma1 = np.array([-1, -.5]), np.array([[1, -.5], [-.5, 2]])\n",
    "mu2, sigma2 = np.array([.5, 1]), np.array([[2, .5], [.5, .5]])\n",
    "z, x = sample_mixture(100, 0.7, mu1, sigma1, mu2, sigma2)\n",
    "test_z, test_x = sample_mixture(100, 0.7, mu1, sigma1, mu2, sigma2)\n",
    "\n",
    "fig = plt.figure(figsize=(8,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Модельные методы классификацииfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "\n",
    "lda = LinearDiscriminantAnalysis(store_covariance=True)\n",
    "z_lda = lda.fit(x, z).predict(x)\n",
    "\n",
    "qda = QuadraticDiscriminantAnalysis(store_covariance=True)\n",
    "z_qda = qda.fit(x, z).predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import colors\n",
    "cmap = colors.LinearSegmentedColormap(\n",
    "    'red_blue_classes',\n",
    "    {'red': [(0, 1, 1), (1, 0.7, 0.7)],\n",
    "     'green': [(0, 0.7, 0.7), (1, 0.7, 0.7)],\n",
    "     'blue': [(0, 0.7, 0.7), (1, 1, 1)]})\n",
    "\n",
    "def plot_ellipse(ax, mu, sigma, color):\n",
    "    v, w = sp.linalg.eigh(sigma)\n",
    "    u = w[0] / sp.linalg.norm(w[0])\n",
    "    angle = np.arctan(u[1] / u[0])\n",
    "    angle = 180 * angle / np.pi\n",
    "    ell = mpl.patches.Ellipse(mu, 2 * v[0] ** 0.5, 2 * v[1] ** 0.5,\n",
    "                              angle=180 + angle, facecolor=color,\n",
    "                              edgecolor='black', linewidth=2)\n",
    "    ell.set_clip_box(ax.bbox)\n",
    "    ell.set_alpha(0.2)\n",
    "    ax.add_artist(ell)\n",
    "    ax.scatter(mu[0], mu[1], marker='+', color=color, s=100)\n",
    "\n",
    "def plot_colormesh(ax, model):\n",
    "    nx, ny = 100, 100\n",
    "    x_min, x_max = plt.xlim()\n",
    "    y_min, y_max = plt.ylim()\n",
    "    xx, yy = np.meshgrid(np.linspace(x_min, x_max, nx), np.linspace(y_min, y_max, ny))\n",
    "    Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])\n",
    "    Z = Z[:, 1].reshape(xx.shape)\n",
    "    plt.pcolormesh(xx, yy, Z, cmap=cmap,\n",
    "                   norm=colors.Normalize(0., 1.), zorder=0)\n",
    "    plt.contour(xx, yy, Z, [0.5], linewidths=2., colors='white')    \n",
    "    plt.contour(xx, yy, Z, [0.2, 0.4, 0.6, 0.8], linewidths=.8, colors='white')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "neigh = KNeighborsClassifier(n_neighbors=10).fit(x, z)\n",
    "z_nn = neigh.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(8,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)\n",
    "# plot_ellipse(ax, lda.means_[0], lda.covariance_, 'r')\n",
    "# plot_ellipse(ax, lda.means_[1], lda.covariance_, 'b')\n",
    "plot_colormesh(ax, neigh)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"LDA priors: %s\" % lda.priors_)\n",
    "\n",
    "fig = plt.figure(figsize=(8,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)\n",
    "plot_ellipse(ax, lda.means_[0], lda.covariance_, 'r')\n",
    "plot_ellipse(ax, lda.means_[1], lda.covariance_, 'b')\n",
    "plot_colormesh(ax, lda)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"QDA priors: %s\" % qda.priors_)\n",
    "\n",
    "fig = plt.figure(figsize=(8,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)\n",
    "plot_ellipse(ax, qda.means_[0], qda.covariance_[0], 'r')\n",
    "plot_ellipse(ax, qda.means_[1], qda.covariance_[1], 'b')\n",
    "plot_colormesh(ax, qda)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Логистическая регрессия"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logregr = linear_model.LogisticRegression(class_weight='balanced')\n",
    "z_logregr = logregr.fit(x, z).predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(8,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)\n",
    "plot_colormesh(ax, logregr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_decision_boundary(ax, model, color, label):\n",
    "    nx, ny = 100, 100\n",
    "    x_min, x_max = plt.xlim()\n",
    "    y_min, y_max = plt.ylim()\n",
    "    xx, yy = np.meshgrid(np.linspace(x_min, x_max, nx), np.linspace(y_min, y_max, ny))\n",
    "    Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])\n",
    "    Z = Z[:, 1].reshape(xx.shape)\n",
    "#     plt.pcolormesh(xx, yy, Z, cmap=cmap,\n",
    "#                    norm=colors.Normalize(0., 1.), zorder=0)\n",
    "#     p = plt.contour(xx, yy, Z, [0.5], linewidths=2., colors='white')    \n",
    "#     plt.contour(xx, yy, Z, [0.2, 0.4, 0.6, 0.8], linewidths=.8, colors='white')    \n",
    "    p = ax.contour(xx, yy, Z, [0.5], linewidths=1.5, colors=[color])\n",
    "    h,l = p.legend_elements()\n",
    "    return h\n",
    "\n",
    "fig = plt.figure(figsize=(10,6))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_points(ax, x, z, mu1, mu2)\n",
    "h1 = plot_decision_boundary(ax, lda, palette[0], \"LDA\")\n",
    "h2 = plot_decision_boundary(ax, qda, palette[1], \"QDA\")\n",
    "h3 = plot_decision_boundary(ax, logregr, palette[2], \"Логистическая регрессия\")\n",
    "plt.legend([h1[0], h2[0], h3[0]], [\"LDA\", \"QDA\", \"Логистическая регрессия\"], loc=\"lower left\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}