{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b8210b19",
   "metadata": {},
   "source": [
    "## A.I. Assignment 5\n",
    "\n",
    "## Learning Goals\n",
    "\n",
    "By the end of this lab, you should be able to:\n",
    "* Get more familiar with tensors in pytorch \n",
    "* Create a simple multilayer perceptron model with pytorch\n",
    "* Visualise the parameters\n",
    "\n",
    "\n",
    "### Task\n",
    "\n",
    "Build a fully connected feed forward network that adds two bits. Determine the a propper achitecture for this network (what database you use for this problem? how many layers? how many neurons on each layer? what is the activation function? what is the loss function? etc)\n",
    "\n",
    "Create at least 3 such networks and compare their performance (how accurate they are?, how farst they are trained to get at 1 accuracy?)\n",
    "\n",
    "Display for the best one the weights for each layer.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e3614e5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from collections import OrderedDict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaf778c4",
   "metadata": {},
   "source": [
    "### 1st Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "5ee7e7d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# your code here\n",
    "model = nn.Sequential(OrderedDict([\n",
    "    ('hidden', nn.Linear(2,3)),\n",
    "     ('sigmoid1', nn.Sigmoid()),\n",
    "     ('output', nn.Linear(3,2)),\n",
    "]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "665ae958",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (hidden): Linear(in_features=2, out_features=3, bias=True)\n",
      "  (sigmoid1): Sigmoid()\n",
      "  (output): Linear(in_features=3, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e26f0d3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [1., 0.],\n",
      "        [1., 1.]])\n"
     ]
    }
   ],
   "source": [
    "# your code here\n",
    "data_in = torch.tensor([[0.,0.],[0.,1.],[1.,0.],[1.,1.]])\n",
    "print(data_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4fb16bbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [0., 1.],\n",
      "        [1., 0.]])\n"
     ]
    }
   ],
   "source": [
    "# your code here\n",
    "data_target = torch.tensor([[0.,0.],[0.,1.],[0.,1.],[1.,0.]])\n",
    "print(data_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "69d920ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# your code here\n",
    "criterion = nn.L1Loss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "cde91f6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Loss = 0.41974443197250366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1001: Loss = 0.3748631477355957\n",
      "Epoch 2001: Loss = 0.37467941641807556\n",
      "Epoch 3001: Loss = 0.37417155504226685\n",
      "Epoch 4001: Loss = 0.37154191732406616\n",
      "Epoch 5001: Loss = 0.3255583643913269\n",
      "Epoch 6001: Loss = 0.19477465748786926\n",
      "tensor([[-0., 0.],\n",
      "        [-0., 1.],\n",
      "        [-0., 1.],\n",
      "        [1., 0.]], grad_fn=<RoundBackward0>)\n",
      "Model reached 100% accuracy at epoch 6449\n"
     ]
    }
   ],
   "source": [
    "# your code here\n",
    "# Train the model\n",
    "for epoch in range(10000):\n",
    "    # reset the gradients to zero before each forward and backward pass\n",
    "    optimizer.zero_grad()  \n",
    "    # forward pass\n",
    "    outputs = model(data_in)      \n",
    "    \n",
    "    # accuracy\n",
    "    predicted = (outputs.round() == data_target)\n",
    "    accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "    if accuracy == 1:\n",
    "        print(outputs.round())\n",
    "        print(f\"Model reached 100% accuracy at epoch {epoch+1}\")\n",
    "        break\n",
    "    # calculate loss\n",
    "    \n",
    "    loss1 = criterion(outputs, data_target)\n",
    "    if epoch % 1000 == 0:\n",
    "        print(f'Epoch {epoch+1}: Loss = {loss1}')\n",
    "    # backward pass\n",
    "    loss1.backward()          \n",
    "    # update weights\n",
    "    optimizer.step()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "dff3ec1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 100.0%\n"
     ]
    }
   ],
   "source": [
    "# your code here\n",
    "# visualize the resuts\n",
    "outputs = model(data_in)\n",
    "predicted = (outputs.round() == data_target)\n",
    "accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "print(f\"accuracy: {accuracy*100}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "c1a7518b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden.weight tensor([[ 3.6313, -3.4167],\n",
      "        [ 1.3280, -0.5874],\n",
      "        [ 0.3637, -2.2525]])\n",
      "hidden.bias tensor([ 2.5554, -0.1830,  0.7138])\n",
      "output.weight tensor([[ 1.7310, -0.1016, -2.1195],\n",
      "        [-2.4398,  1.7537,  0.6634]])\n",
      "output.bias tensor([-0.1873,  1.1474])\n"
     ]
    }
   ],
   "source": [
    "# your code here\n",
    "# print model wights\n",
    "for name, param in model.named_parameters():\n",
    "    print(name, param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b569b90",
   "metadata": {},
   "source": [
    "### 2nd Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "4cdf09ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (hidden1): Linear(in_features=2, out_features=3, bias=True)\n",
      "  (sigmoid1): Sigmoid()\n",
      "  (hidden2): Linear(in_features=3, out_features=4, bias=True)\n",
      "  (sigmoid2): Sigmoid()\n",
      "  (output1): Linear(in_features=4, out_features=2, bias=True)\n",
      "  (relu): ReLU()\n",
      ")\n",
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [1., 0.],\n",
      "        [1., 1.]])\n",
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [0., 1.],\n",
      "        [1., 0.]])\n",
      "tensor([[0.0000, 0.0000],\n",
      "        [0.0000, 1.0630],\n",
      "        [0.0000, 1.0580],\n",
      "        [0.0000, 0.0000]], grad_fn=<ReluBackward0>)\n",
      "accuracy: 87.5%\n",
      "hidden1.weight tensor([[3.7943, 3.9107],\n",
      "        [1.3786, 1.1741],\n",
      "        [0.5741, 0.9622]])\n",
      "hidden1.bias tensor([-0.3776, -1.5335, -0.2704])\n",
      "hidden2.weight tensor([[-2.0642,  1.3785,  0.4542],\n",
      "        [ 2.6336, -1.9812, -1.4470],\n",
      "        [-0.2936, -0.5167,  0.0780],\n",
      "        [-1.2268,  1.2764,  0.6379]])\n",
      "hidden2.bias tensor([ 0.4410, -0.3201, -0.1641, -0.3203])\n",
      "output1.weight tensor([[ 0.0149,  0.3398, -0.2382,  0.0658],\n",
      "        [-2.4774,  3.5136,  0.1576, -1.6128]])\n",
      "output1.bias tensor([-0.2175,  0.2636])\n"
     ]
    }
   ],
   "source": [
    "model = nn.Sequential(OrderedDict([\n",
    "    ('hidden1', nn.Linear(2,3)),\n",
    "     ('sigmoid1', nn.Sigmoid()),\n",
    "     ('hidden2', nn.Linear(3,4)),\n",
    "     ('sigmoid2', nn.Sigmoid()),\n",
    "     ('output1', nn.Linear(4,2)),\n",
    "     ('relu', nn.ReLU()),\n",
    "]))\n",
    "print(model)\n",
    "data_in = torch.tensor([[0.,0.],[0.,1.],[1.,0.],[1.,1.]])\n",
    "print(data_in)\n",
    "data_target = torch.tensor([[0.,0.],[0.,1.],[0.,1.],[1.,0.]])\n",
    "print(data_target)\n",
    "criterion = nn.L1Loss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "for epoch in range(10000):\n",
    "    # reset the gradients to zero before each forward and backward pass\n",
    "    optimizer.zero_grad()  \n",
    "    # forward pass\n",
    "    outputs = model(data_in)        \n",
    "    \n",
    "    # accuracy\n",
    "    predicted = (outputs.round() == data_target)\n",
    "    accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "    if accuracy == 1:\n",
    "        print(f\"Model 1 reached 100% accuracy at epoch {epoch+1}\")\n",
    "        break\n",
    "    # calculate loss\n",
    "    loss1 = criterion(outputs, data_target)\n",
    "    # backward pass\n",
    "    loss1.backward()          \n",
    "    # update weights\n",
    "    optimizer.step() \n",
    "outputs = model(data_in)\n",
    "print(outputs)\n",
    "predicted = (outputs.round() == data_target)\n",
    "accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "print(f\"accuracy: {accuracy*100}%\")\n",
    "for name, param in model.named_parameters():\n",
    "    print(name, param.data)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d95ac7a",
   "metadata": {},
   "source": [
    "### 3th Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d0bea66c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (hidden1): Linear(in_features=2, out_features=3, bias=True)\n",
      "  (relu1): ReLU()\n",
      "  (output1): Linear(in_features=3, out_features=2, bias=True)\n",
      ")\n",
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [1., 0.],\n",
      "        [1., 1.]])\n",
      "tensor([[0., 0.],\n",
      "        [0., 1.],\n",
      "        [0., 1.],\n",
      "        [1., 0.]])\n",
      "Epoch 1: Loss = 0.3959614038467407\n",
      "Epoch 101: Loss = 0.11625192314386368\n",
      "Epoch 201: Loss = 0.08679977804422379\n",
      "Epoch 301: Loss = 0.06458555907011032\n",
      "Epoch 401: Loss = 0.06256230920553207\n",
      "Epoch 501: Loss = 0.06250278651714325\n",
      "Epoch 601: Loss = 0.06250019371509552\n",
      "Epoch 701: Loss = 0.0625000074505806\n",
      "Epoch 801: Loss = 0.0625\n",
      "Epoch 901: Loss = 0.0625\n",
      "accuracy: 87.5%\n",
      "hidden1.weight tensor([[-0.0102,  0.3543],\n",
      "        [ 0.3771,  1.0492],\n",
      "        [-0.8659,  0.8689]])\n",
      "hidden1.bias tensor([-0.5027, -0.3777, -0.0030])\n",
      "output1.weight tensor([[-0.5325,  0.9537, -0.7395],\n",
      "        [ 0.1828, -0.4768,  0.9471]])\n",
      "output1.bias tensor([-2.4464e-07,  5.0000e-01])\n"
     ]
    }
   ],
   "source": [
    "model = nn.Sequential(OrderedDict([\n",
    "    ('hidden1', nn.Linear(2,3)),\n",
    "     ('relu1', nn.ReLU()),\n",
    "     ('output1', nn.Linear(3,2)),\n",
    "]))\n",
    "print(model)\n",
    "data_in = torch.tensor([[0.,0.],[0.,1.],[1.,0.],[1.,1.]])\n",
    "print(data_in)\n",
    "data_target = torch.tensor([[0.,0.],[0.,1.],[0.,1.],[1.,0.]])\n",
    "print(data_target)\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "for epoch in range(1000):\n",
    "    # reset the gradients to zero before each forward and backward pass\n",
    "    optimizer.zero_grad()  \n",
    "    # forward pass\n",
    "    outputs = model(data_in)        \n",
    "    \n",
    "    # accuracy\n",
    "    predicted = (outputs.round() == data_target)\n",
    "    accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "    if accuracy == 1:\n",
    "        print(f\"Model 1 reached 100% accuracy at epoch {epoch+1}\")\n",
    "        break\n",
    "    # calculate loss\n",
    "    loss1 = criterion(outputs, data_target)\n",
    "    if epoch % 100 == 0:\n",
    "        print(f'Epoch {epoch+1}: Loss = {loss1}')\n",
    "    # backward pass\n",
    "    loss1.backward()          \n",
    "    # update weights\n",
    "    optimizer.step() \n",
    "outputs = model(data_in)\n",
    "predicted = (outputs.round() == data_target)\n",
    "accuracy = predicted.sum().item() / (data_target == data_target).sum().item()\n",
    "print(f\"accuracy: {accuracy*100}%\")\n",
    "for name, param in model.named_parameters():\n",
    "    print(name, param.data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e29c65a2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}