WIP multi nodes

Author: nktpro

.envrc

@@ -1 +1,2 @@
 use flake
+if [[ -f "./.env" ]]; then source "./.env"; fi

.gitignore

@@ -1,4 +1,5 @@
 /.vscode
 /.zsh_history
 .ssh
-/result
+/result
+/.env

deno.lock

@@ -0,0 +1,4 @@
+{
+  "url": "https://192.168.2.152:6443",
+  "token": "K103daab472e6b01fe590a22d779bcf6198da3a036849f7ca175c8f717b2ad66684::server:114d6916eec0ed6c94cc2c32bf4602cb"
+}

examples/multi/.secrets/join.json

@@ -0,0 +1,32 @@
+import { ServerInstanceConfig } from "../../src/types.ts";
+
+export default {
+  name: "jetski-multi-node-one",
+  image: "22.04",
+  cpus: 1,
+  memoryGiBs: 2,
+  diskGiBs: 5,
+  k3sVersion: "v1.24.17+k3s1",
+  serviceCidr: "10.254.251.0/24",
+  clusterCidr: "10.254.252.0/22",
+  clusterDnsIp: "10.254.255.10",
+  clusterDomain: "jetski.local",
+  bridged: Boolean(Deno.env.get("JETSKI_INSTANCE_BRIDGED")),
+  externalNetworkCidr: Deno.env.get("JETSKI_INSTANCE_NODE_IP_CIDR"),
+  externalNetworkInterface: "eth1",
+  disableComponents: {
+    traefik: true,
+    metricsServer: true
+  },
+  nodeLabels: {
+    "com.jetski/foo": "bar",
+    "com.jetski/baz": "boo",
+  },
+  // datastoreEndpoint: "http://192.168.2.22:2379"
+  kubelet: {
+    maxPods: 500
+  },
+  isBootstrapInstance: true,
+  sshDirectoryPath: "./.secrets/.ssh",
+  joinMetadataPath: "./.secrets/join.json",
+} satisfies ServerInstanceConfig;

examples/multi/multi-one.ts

@@ -0,0 +1,16 @@
+import { ServerInstanceConfig } from "../../src/types.ts";
+import bootstrapInstanceConfig from "./multi-one.ts";
+
+export default {
+  ...bootstrapInstanceConfig,
+  name: "jetski-multi-node-two",
+  image: "22.04",
+  cpus: 1,
+  memoryGiBs: 2,
+  diskGiBs: 5,
+  nodeLabels: {
+    "com.jetski/foo": "bar",
+    "com.jetski/baz": "boo",
+  },
+  isBootstrapInstance: false
+} satisfies ServerInstanceConfig;

examples/multi/multi-two.ts

@@ -1,7 +1,7 @@
-import { InstanceConfig } from "../src/types.ts";
+import { ServerInstanceConfig } from "../../src/types.ts";
 
 export default {
-  name: "jetski-example",
+  name: "jetski-single-node",
   image: "22.04",
   cpus: 1,
   memoryGiBs: 2,
@@ -11,8 +11,8 @@ export default {
   serviceCidr: "10.254.255.0/24",
   clusterDnsIp: "10.254.255.10",
   clusterDomain: "jetski.local",
-  bridged: true,
-  filterSshIpByCidr: "192.168.2.0/24",
+  bridged: Boolean(Deno.env.get("JETSKI_INSTANCE_BRIDGED")),
+  externalNetworkCidr: Deno.env.get("JETSKI_INSTANCE_FILTER_SSH_IP_BY_CIDR"),
   disableComponents: {
     traefik: true,
     metricsServer: true
@@ -21,9 +21,9 @@ export default {
     "com.jetski/foo": "bar",
     "com.jetski/baz": "boo",
   },
-  sshDirectoryPath: "./local/.ssh",
+  sshDirectoryPath: "./.ssh",
   // datastoreEndpoint: "http://192.168.2.22:2379"
   kubelet: {
     maxPods: 500
   }
-} as InstanceConfig;
+} satisfies ServerInstanceConfig;

examples/local.ts examples/single/single.ts

@@ -80,13 +80,13 @@ export async function updateKubeconfig(
   ok("Kubeconfig has been updated. The current context should now be", cyan(name));
 }
 
-export async function createInstance(instance: InstanceConfig) {
+export async function createInstance(instance: InstanceConfig, signal: AbortSignal) {
   const { sshDirectoryPath, cpus, memoryGiBs, diskGiBs, image, name, bridged } = instance;
 
   await generateSshKeyPairIfNotExists(instance);
 
   const sshPublicKey = await getSshPublicKey(sshDirectoryPath);
-  const cloudInitConfig = createCloudInitConfig({ sshPublicKey, instance });
+  const cloudInitConfig = await createCloudInitConfig({ sshPublicKey, instance });
   const tempDir = await Deno.makeTempDir();
   const cloudInitFilePath = joinPath(tempDir, "cloud-init.yaml");
   log("Generated cloud-init.yaml");
@@ -163,20 +163,22 @@ export async function createInstance(instance: InstanceConfig) {
     // Ignore
   }
 
-  const ip = await multipassPostStart(instance);
+  const ip = await multipassPostStart(instance, signal);
 
-  await updateKubeconfig({ ip, instance });
+  if (instance.isBootstrapInstance) {
+    await updateKubeconfig({ ip, instance });
+  }
 }
 
 export default createCliAction(
   Type.Object({
     config: InstanceConfigPathSchema,
   }),
-  async ({ config: configPath }) => {
+  async ({ config: configPath }, _, signal) => {
     const absoluteConfigPath = resolvePath(configPath);
     const instance = await loadInstanceConfig(absoluteConfigPath);
 
-    await createInstance(instance);
+    await createInstance(instance, signal);
 
     return ExitCode.Zero;
   },

src/actions/create.ts

@@ -8,8 +8,12 @@ export async function destroyInstance(instance: InstanceConfig) {
   const { state, ipv4 } = await multipassInfo(instance);
 
   if (state === InstanceState.Running) {
-    const ip = getSshIp(ipv4, instance.filterSshIpByCidr);
-    await multipassUnroute({ ip, instance });
+    const ip = getSshIp(ipv4, instance.externalNetworkCidr);
+
+    if (instance.isBootstrapInstance) {
+      await multipassUnroute({ ip, instance });
+    }
+
     await multipassK3sKillAll({ ip, sshDirectoryPath });
     await multipassStop(instance);
   }

src/actions/destroy.ts

@@ -8,7 +8,7 @@ export default createCliAction(
   Type.Object({
     config: InstanceConfigPathSchema,
   }),
-  async ({ config: configPath }) => {
+  async ({ config: configPath }, _, signal) => {
     const absoluteConfigPath = resolvePath(configPath);
     const instance = await loadInstanceConfig(absoluteConfigPath);
     const { name } = instance;
@@ -18,7 +18,7 @@ export default createCliAction(
       throw new Error(`Instance '${name}' is not in 'Running' state. Current state is '${state}'`);
     }
 
-    const ip = await multipassPostStart(instance);
+    const ip = await multipassPostStart(instance, signal);
     await updateKubeconfig({ ip, instance });
 
     ok(`Local routes and kubeconfig for instance '${name}' have been updated with IP: ${ip}`);

src/actions/refresh.ts

@@ -11,7 +11,7 @@ export default createCliAction(
       default: false,
     })),
   }),
-  async ({ config: configPath, skipConfirm }) => {
+  async ({ config: configPath, skipConfirm }, _, signal) => {
     const absoluteConfigPath = resolvePath(configPath);
     const instance = await loadInstanceConfig(absoluteConfigPath);
     const { name } = instance;
@@ -28,7 +28,7 @@ export default createCliAction(
     }
 
     await destroyInstance(instance);
-    await createInstance(instance);
+    await createInstance(instance, signal);
 
     return ExitCode.Zero;
   },

src/actions/reset.ts

@@ -19,10 +19,13 @@ export default createCliAction(
       throw new Error(`Instance '${name}' is not in 'Running' state. Current state is '${state}'`);
     }
 
+    const ip = getSshIp(ipv4, instance.externalNetworkCidr);
+    log(gray(`Instance IP is '${ip}'`));
+
     const exitCode = await multipassSshInteractive({
       cmd: unparsedArgs,
       sshDirectoryPath,
-      ip: getSshIp(ipv4, instance.filterSshIpByCidr),
+      ip: getSshIp(ipv4, instance.externalNetworkCidr),
     });
 
     return new ExitCode(exitCode);

src/actions/ssh.ts

@@ -8,7 +8,7 @@ export default createCliAction(
   Type.Object({
     config: InstanceConfigPathSchema,
   }),
-  async ({ config: configPath }) => {
+  async ({ config: configPath }, _, signal) => {
     const absoluteConfigPath = resolvePath(configPath);
     const instance = await loadInstanceConfig(absoluteConfigPath);
     const { name } = instance;
@@ -21,8 +21,11 @@ export default createCliAction(
     }
 
     await multipassStart(instance);
-    const ip = await multipassPostStart(instance);
-    await updateKubeconfig({ ip, instance });
+    const ip = await multipassPostStart(instance, signal);
+
+    if (instance.isBootstrapInstance) {
+      await updateKubeconfig({ ip, instance });
+    }
 
     ok(`Instance '${name}' has been started`);
 

src/actions/start.ts

@@ -18,9 +18,11 @@ export default createCliAction(
       throw new Error(`Instance '${name}' is not in 'Running' state. Current state is '${state}'`);
     }
 
-    const ip = getSshIp(ipv4, instance.filterSshIpByCidr);
+    const ip = getSshIp(ipv4, instance.externalNetworkCidr);
 
-    await multipassUnroute({ ip, instance });
+    if (instance.isBootstrapInstance) {
+      await multipassUnroute({ ip, instance });
+    }
     await multipassK3sKillAll({ ip, sshDirectoryPath });
     await multipassStop(instance);
 

src/actions/stop.ts

@@ -18,9 +18,11 @@ export default createCliAction(
       throw new Error(`Instance '${name}' is not in 'Running' state. Current state is '${state}'`);
     }
 
-    const ip = getSshIp(ipv4, instance.filterSshIpByCidr);
+    const ip = getSshIp(ipv4, instance.externalNetworkCidr);
 
-    await multipassUnroute({ ip, instance });
+    if (instance.isBootstrapInstance) {
+      await multipassUnroute({ ip, instance });
+    }
     await multipassSuspend(instance);
 
     ok(`Instance '${name}' has been suspended`);

src/actions/suspend.ts

@@ -1,8 +1,8 @@
-export * from "https://deno.land/x/utils@2.15.2/cli_utils.ts";
-export * from "https://deno.land/x/utils@2.15.2/exec_utils.ts";
-export * from "https://deno.land/x/utils@2.15.2/deps/typebox.ts";
-export * from "https://deno.land/x/[email protected]/validation_utils.ts";
-export type { ValidationResult } from "https://deno.land/x/utils@2.15.2/validation_utils.ts";
+export * from "https://deno.land/x/utils@2.17.0/cli_utils.ts";
+export * from "https://deno.land/x/utils@2.17.0/exec_utils.ts";
+export * from "https://deno.land/x/utils@2.17.0/validation_utils.ts";
+export * from "./deps/typebox.ts";
+export type { ValidationResult } from "https://deno.land/x/utils@2.17.0/validation_utils.ts";
 export * from "https://deno.land/[email protected]/fmt/colors.ts";
 
 export {
@@ -18,7 +18,8 @@ export { parse as parseYaml, stringify as stringifyYaml } from "https://deno.lan
 export type { YAMLError } from "https://deno.land/[email protected]/yaml/_error.ts";
 
 export { exists as fsExists } from "https://deno.land/[email protected]/fs/exists.ts";
+export { ensureFile } from "https://deno.land/[email protected]/fs/ensure_file.ts";
 export { assertExists } from "https://deno.land/[email protected]/assert/assert_exists.ts";
-export { memoizePromise } from "https://deno.land/x/utils@2.15.2/async_utils.ts";
+export { memoizePromise } from "https://deno.land/x/utils@2.17.0/async_utils.ts";
 
 export { delay } from "https://deno.land/[email protected]/async/delay.ts";

src/deps.ts

@@ -0,0 +1,2 @@
+export * from "https://deno.land/x/[email protected]/deps/typebox.ts";
+export * from "https://deno.land/x/[email protected]/typebox_utils.ts";