Switch from logit to linear-sum

Author: Atcold

04-spiral_classification.ipynb

@@ -132,7 +132,7 @@
     ")\n",
     "model.to(device)  # possibly send to CUDA\n",
     "\n",
-    "# Cross entropy given the output logits\n",
+    "# Cross entropy given the linear output\n",
     "C = torch.nn.CrossEntropyLoss(reduction='none')\n",
     "\n",
     "# Using Adam optimiser\n",
@@ -141,11 +141,11 @@
     "# Full-batch training loop\n",
     "for t in range(2_000):\n",
     "    \n",
-    "    # Feed forward to get the logits\n",
-    "    l = model(X)\n",
+    "    # Feed forward to get the linear sum s\n",
+    "    s = model(X)\n",
     "    \n",
-    "    # Compute the free energy F\n",
-    "    F = C(l, y)\n",
+    "    # Compute the free energy F and loss L\n",
+    "    F = C(s, y)\n",
     "    L = F.mean()\n",
     "    \n",
     "    # Zero the gradients\n",
@@ -159,7 +159,7 @@
     "    optimiser.step()\n",
     "    \n",
     "    # Display epoch, L, and accuracy\n",
-    "    overwrite(f'[EPOCH]: {t}, [LOSS]: {L.item():.6f}, [ACCURACY]: {acc(l, y):.3f}')"
+    "    overwrite(f'[EPOCH]: {t}, [LOSS]: {L.item():.6f}, [ACCURACY]: {acc(s, y):.3f}')"
    ]
   },
   {
@@ -189,13 +189,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Compute logits for a fine grid over the input space\n",
+    "# Compute linear output s for a fine grid over the input space\n",
     "\n",
     "mesh = torch.arange(-1.5, 1.5, 0.01)\n",
     "xx, yy = torch.meshgrid(mesh, mesh)\n",
     "grid = torch.stack((xx.reshape(-1), yy.reshape(-1)), dim=1)\n",
     "with torch.no_grad():\n",
-    "    logits = model(grid)"
+    "    s = model(grid)"
    ]
   },
   {
@@ -207,7 +207,7 @@
     "# Choice of free energy\n",
     "\n",
     "fe = 'cross-entropy'\n",
-    "fe = 'negative logit'"
+    "fe = 'negative linear output'"
    ]
   },
   {
@@ -242,12 +242,12 @@
     "\n",
     "for k in range(K):\n",
     "    if fe == 'cross-entropy':\n",
-    "        F = C(logits, torch.LongTensor(1).fill_(k).expand(logits.size(0)))\n",
+    "        F = C(s, torch.LongTensor(1).fill_(k).expand(s.size(0)))\n",
     "        F = F.reshape(xx.shape)\n",
     "        plot_2d_energy_levels(X, y, (xx, yy, F, k, K), (0, 35), (1, 35, 4))\n",
     "\n",
-    "    elif fe == 'negative logit':\n",
-    "        F = -logits[:, k]\n",
+    "    elif fe == 'negative linear output':\n",
+    "        F = -s[:, k]\n",
     "        F = F.reshape(xx.shape)\n",
     "        plot_2d_energy_levels(X, y, (xx, yy, F, k, K), (-20, 20), (-20, 21, 2.5))\n",
     "        \n",
@@ -282,7 +282,7 @@
     "# Cross-entropy\n",
     "if fe == 'cross-entropy':\n",
     "    fig, ax = plot_3d_energy_levels(X, y, (xx, yy, F, k, K), (0, 18), (0, 19, 1), (0, 19, 2))\n",
-    "elif fe == 'negative logit':\n",
+    "elif fe == 'negative linear output':\n",
     "    fig, ax = plot_3d_energy_levels(X, y, (xx, yy, F, k, K), (-30, 20), (-30, 20, 1), (-30, 21, 5))"
    ]
   },
@@ -336,7 +336,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,