overview: render service demos (#668)

Author: erictapen

flake.nix

@@ -119,7 +119,13 @@
         rec {
           packages = ngipkgs // {
             overview = import ./overview {
-              inherit lib lib' self;
+              inherit
+                lib
+                lib'
+                self
+                nixpkgs
+                system
+                ;
               pkgs = pkgs // ngipkgs;
               projects = ngiProjects;
               options = optionsDoc.optionsNix;
@@ -191,7 +197,8 @@
 
           devShells.default = pkgs.mkShell {
             inherit (checks."infra/pre-commit") shellHook;
-            buildInputs = checks."infra/pre-commit".enabledPackages;
+            # TODO use devmode
+            buildInputs = checks."infra/pre-commit".enabledPackages ++ [ pkgs.darkhttpd ];
           };
 
           formatter = pkgs.writeShellApplication {

overview/default.nix

@@ -2,9 +2,11 @@
   lib,
   lib',
   options,
+  nixpkgs,
   pkgs,
   projects,
   self,
+  system,
 }:
 let
   inherit (builtins)
@@ -14,6 +16,7 @@ let
     concatStringsSep
     filter
     isList
+    isInt
     readFile
     substring
     toJSON
@@ -30,17 +33,35 @@ let
     mapAttrsToList
     optionalString
     recursiveUpdate
+    filterAttrs
     mapAttrs'
     nameValuePair
     take
     drop
+    splitString
+    intersperse
     ;
 
   empty =
     xs:
     assert isList xs;
     xs == [ ];
-  heading = i: text: "<h${toString i}>${text}</h${toString i}>";
+  heading =
+    i: anchor: text:
+    assert (isInt i && i > 0);
+    if i == 1 then
+      ''
+        <h1>${text}</h1>
+      ''
+    else
+      ''
+        <a class="heading" href="#${anchor}">
+          <h${toString i} data-url="#${anchor}">
+            ${text}
+            <span class="anchor"/>
+          </h${toString i}>
+        </a>
+      '';
 
   # Splits a compressed date up into ISO 8601
   lastModified =
@@ -71,7 +92,7 @@ let
         );
       in
       filter (option: any ((flip hasPrefix) (join "." option.loc)) spec) (attrValues options);
-    examples = project: attrValues project.nixos.examples;
+    examples = project: attrValues (filterAttrs (name: _: name != "demo") project.nixos.examples);
   };
 
   # This doesn't actually produce a HTML string but a Jinja2 template string
@@ -116,28 +137,26 @@ let
           prefixLength = 2;
         in
         optionalString (!empty projectOptions) ''
-          <section><details><summary>${heading 3 "Options"}</summary><dl>
+          ${heading 2 "options" "Options"}
+          <section><details><summary><code>services.cryptpad</code></summary><dl>
           ${concatLines (map (one prefixLength) projectOptions)}
           </dl></details></section>
         '';
     };
 
     examples = rec {
       one = example: ''
-        <li>
+        <section><details><summary>${example.description}</summary>
 
-        ${example.description}
+        {{ include_code("nix", "${example.module}")}}
 
-        <pre><code>${readFile example.module}</code></pre>
-
-        </li>
+        </details></section>
       '';
       many =
         examples:
         optionalString (!empty examples) ''
-          <section><details><summary>${heading 3 "Examples"}</summary><ul>
+          ${heading 2 "examples" "Examples"}
           ${concatLines (map one examples)}
-          </ul></details></section>
         '';
     };
 
@@ -176,8 +195,11 @@ let
     # The indivdual page of a project
     projects.one = name: project: ''
       <article class="page-width">
-        ${heading 1 name}
+        ${heading 1 null name}
         ${render.metadata.one project.metadata}
+        ${optionalString (project.nixos.examples ? demo) (
+          render.serviceDemo.one project.nixos.modules.services project.nixos.examples.demo
+        )}
         ${render.options.many (pick.options project)}
         ${render.examples.many (pick.examples project)}
       </article>
@@ -221,12 +243,56 @@ let
         '';
       many = projects: concatLines (mapAttrsToList one projects);
     };
+
+    demoGlue.one = exampleText: ''
+      # default.nix
+      {
+        ngipkgs ? import (fetchTarball "https://github.com/ngi-nix/ngipkgs/tarball/main") { },
+      }:
+      ngipkgs.demo (
+        ${toString (intersperse "\n " (splitString "\n" exampleText))}
+      )
+    '';
+
+    serviceDemo.one =
+      services: example:
+      let
+        demoSystem = import (nixpkgs + "/nixos/lib/eval-config.nix") {
+          inherit system;
+          modules = (attrValues services) ++ [ example.module ];
+        };
+        openPorts = demoSystem.config.networking.firewall.allowedTCPPorts;
+        # The port that is forwarded to the host so that the user can access the demo service.
+        servicePort = (builtins.head openPorts) + 10000;
+      in
+      ''
+        ${heading 2 "demo" "Run a demo deployment locally"}
+
+        <ol>
+          <li><strong>Install Nix on your platform.</strong></li>
+          <li>
+            <strong>Download this Nix file to your computer.</strong>
+            It obtains the NGIpkgs source code and declares a basic service configuration
+            to be run in a virtual machine.
+            {{ include_code("nix", "default.nix", relative_path=True) }}
+          </li>
+          <li>
+            <strong>Build the virtual machine</strong> defined in <code>default.nix</code> and run it:
+            <pre><code>nix-build && ./result</code></pre>
+            Building <strong>will</strong> take a while.
+          </li>
+          <li>
+            <strong>Access the service</strong> with a web browser:
+            <a href="http://localhost:${toString servicePort}">http://localhost:${toString servicePort}</a>
+          </li>
+        </ol>
+      '';
   };
 
   # The top-level overview for all projects
   index = ''
     <section class="page-width">
-      ${heading 1 "NGIpkgs"}
+      ${heading 1 null "NGIpkgs"}
 
       <p>
         NGIpkgs is collection of software applications funded by the <a href="https://www.ngi.eu/ngi-projects/ngi-zero/">Next Generation Internet</a> initiative and packaged for <a href="https://nixos.org">NixOS</a>.
@@ -267,26 +333,30 @@ let
     <footer>Version: ${version}, Last Modified: ${lastModified}</footer>
   '';
 
-  # Every HTML page that we generate
-  pages =
-    {
-      "index.html" = {
-        pagetitle = "NGIpkgs software repository";
-        content = index;
-        summary = ''
-          NGIpkgs is collection of software applications funded by the Next
-          Generation Internet initiative and packaged for NixOS. 
-        '';
-      };
+  # HTML project pages
+  projectPages = mapAttrs' (
+    name: project:
+    nameValuePair "project/${name}" {
+      pagetitle = "NGIpkgs | ${name}";
+      content = render.projects.one name project;
+      summary = project.metadata.summary or null;
+      demoFile =
+        if project.nixos.examples ? demo then
+          (pkgs.writeText "default.nix" (render.demoGlue.one (readFile project.nixos.examples.demo.module)))
+        else
+          null;
     }
-    // mapAttrs' (
-      name: project:
-      nameValuePair "project/${name}/index.html" {
-        pagetitle = "NGIpkgs | ${name}";
-        content = render.projects.one name project;
-        summary = project.metadata.summary or null;
-      }
-    ) projects;
+  ) projects;
+
+  # The summary page at the overview root
+  indexPage = {
+    pagetitle = "NGIpkgs software repository";
+    content = index;
+    summary = ''
+      NGIpkgs is collection of software applications funded by the Next
+      Generation Internet initiative and packaged for NixOS. 
+    '';
+  };
 
   htmlFile =
     path:
@@ -313,10 +383,19 @@ let
     '';
 
   # Ensure that directories exist and render the jinja2 template that we composed with Nix so far
-  writeHtmlCommand = path: htmlFile: ''
-    mkdir -p "$out/$(dirname '${path}')"
-    python3 ${./render-template.py} '${htmlFile}' "$out/${path}"
-  '';
+  writeProjectCommand =
+    path: page:
+    ''
+      mkdir -p "$out/${path}"
+    ''
+    + optionalString (page.demoFile != null) ''
+      cp '${page.demoFile}' "$out/${path}/default.nix"
+      chmod +w "$out/${path}/default.nix"
+      nixfmt "$out/${path}/default.nix"
+    ''
+    + ''
+      python3 ${./render-template.py} '${htmlFile path page}' "$out/${path}/index.html"
+    '';
 
   fonts =
     pkgs.runCommand "fonts"
@@ -331,6 +410,12 @@ let
         done
       '';
 
+  highlightingCss =
+    pkgs.runCommand "pygments-css-rules.css" { nativeBuildInputs = [ pkgs.python3Packages.pygments ]; }
+      ''
+        pygmentize -S default -f html -a .code > $out
+      '';
+
 in
 pkgs.runCommand "overview"
   {
@@ -341,17 +426,20 @@ pkgs.runCommand "overview"
         ps: with ps; [
           jinja2
           markdown-it-py
+          pygments
         ]
       ))
+      nixfmt-rfc-style
     ];
   }
   (
     ''
       mkdir -pv $out
-      cp -v ${./style.css} $out/style.css
+      cat ${./style.css} ${highlightingCss} > $out/style.css
       ln -s ${fonts} $out/fonts
+      python3 ${./render-template.py} '${htmlFile "" indexPage}' "$out/index.html"
     ''
-    + (concatLines (mapAttrsToList (path: v: writeHtmlCommand path (htmlFile path v)) pages))
+    + (concatLines (mapAttrsToList (path: page: writeProjectCommand path page) projectPages))
     + ''
 
       vnu -Werror --format json $out/*.html | jq

overview/render-template.py

@@ -1,16 +1,35 @@
 import sys
+import os
+
 from jinja2 import Template
 from markdown_it import MarkdownIt
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.formatters import HtmlFormatter
 
 jinja2_template_file = sys.argv[1]
 output_file = sys.argv[2]
 
+# Include code from a file and highlight it in HTML
+def include_code(language, file_path, relative_path=False):
+  if relative_path:
+    file_path = os.path.dirname(output_file) + "/" + file_path
+  with open(file_path) as file:
+    code = file.read()
+    try:
+        lexer = get_lexer_by_name(language, stripall=True)
+        formatter = HtmlFormatter(cssclass="code")
+        return highlight(code, lexer, formatter)
+    except Exception:
+        print("Fallback: Return the code without highlighting")
+        return f'<pre><code>{code}</code></pre>'
+
 md = MarkdownIt("commonmark")
 
 with open(jinja2_template_file) as file:
   jinja2_template = Template(file.read())
 
-rendered = jinja2_template.render(markdown_to_html=md.render)
+rendered = jinja2_template.render(markdown_to_html=md.render, include_code=include_code)
 
 with open(output_file, "w") as file:
   file.write(rendered)

overview/style.css

@@ -274,6 +274,7 @@
 
 body {
     font-family: "IBM Plex Sans";
+    padding-bottom: 25em;
 }
 body > section {
     padding: 0.5cm 0 0.5cm 0;
@@ -282,6 +283,7 @@ body > section {
 code, pre {
   font-family: "IBM Plex Mono";
   background: #eee;
+  overflow-x: auto;
 }
 
 details > summary > h2, details > summary > h3 {
@@ -293,6 +295,24 @@ h1 {
   font-weight: 700;
 }
 
+a.heading {
+  text-decoration: none;
+  color: inherit;
+}
+
+h2:hover .anchor {
+    visibility: visible;
+    opacity: 1;
+}
+
+.anchor {
+    visibility: hidden;
+    opacity: 0;
+    transition: opacity 0.3s ease;
+    font-size: 75%;
+    content: "¶";
+}
+
 .page-width {
   max-width: 55rem;
   margin: auto;
@@ -330,7 +350,7 @@ article.project .deliverable-tag {
 
 .option-name {
   font-weight: 600;
-  font-family: "IBMPlexMono";
+  font-family: "IBM Plex Mono";
 }
 
 .option-readonly {