feat!: binary patching alternative (#112)

Author: LouisHaftmann

.dockerignore

@@ -7,4 +7,5 @@ node_modules/
 data/
 .env*
 .eslintcache
-test/
+test/
+certs/

.env

@@ -1,5 +1,7 @@
 API_BASE_URL=http://localhost:3000
 DEBUG=true
+CA_KEY_PATH=./certs/key.pem
+CA_CERT_PATH=./certs/cert.pem
 
 # filesystem
 STORAGE_DRIVER=filesystem

.github/workflows/ci-cd.yml

@@ -50,7 +50,16 @@ jobs:
       - name: pnpm install
         uses: falcondev-it/.github/actions/pnpm-install@master
 
-      - run: VITEST_DB_DRIVER=${{ matrix.db-driver }} VITEST_STORAGE_DRIVER=${{ matrix.storage-driver }} pnpm run test:run
+      - run: touch mock.key
+      - run: touch mock.crt
+
+      - run: pnpm run test:run
+        env:
+          VITEST_DB_DRIVER: ${{ matrix.db-driver }}
+          VITEST_STORAGE_DRIVER: ${{ matrix.storage-driver }}
+          DISABLE_PROXY: true
+          CA_CERT_PATH: mock.crt
+          CA_KEY_PATH: mock.key
 
   deploy:
     if: github.event.ref == 'refs/heads/dev'

.gitignore

@@ -10,4 +10,6 @@ data/
 .data
 .idea
 .DS_Store
-tests/temp/
+tests/temp/
+certs/*
+!certs/.gitkeep

README.md

@@ -10,20 +10,30 @@ This is a drop-in replacement for the official GitHub hosted cache server. It is
 - 😎 Easy setup
 
 ```yaml
-version: '3.9'
-
 services:
   cache-server:
-    image: ghcr.io/falcondev-oss/github-actions-cache-server
+    image: ghcr.io/falcondev-oss/github-actions-cache-server:latest
     ports:
       - '3000:3000'
+      - '8000:8000'
     environment:
       API_BASE_URL: http://localhost:3000
+      CA_KEY_PATH: /run/secrets/ca_key
+      CA_CERT_PATH: /run/secrets/ca_cert
     volumes:
       - cache-data:/app/.data
+    secrets:
+      - ca_key
+      - ca_cert
 
 volumes:
   cache-data:
+
+secrets:
+  ca_key:
+    file: ./key.pem
+  ca_cert:
+    file: ./cert.pem
 ```
 
 ## Documentation

certs/.gitkeep

@@ -30,7 +30,7 @@ export default defineAppConfig({
     ],
   },
   footer: {
-    credits: 'Copyright © 2024',
+    credits: 'Made wih ❤️ by @falcondev-oss',
     colorMode: false,
     links: [
       {

docs/app.config.ts

@@ -8,20 +8,30 @@ The cache server comes as a Docker image and can be deployed using Docker Compos
 ## 1. Deploying the Cache Server
 
 ```yaml [docker-compose.yml]
-version: '3.9'
-
 services:
   cache-server:
     image: ghcr.io/falcondev-oss/github-actions-cache-server:latest
     ports:
       - '3000:3000'
+      - '8000:8000'
     environment:
       API_BASE_URL: http://localhost:3000
+      CA_KEY_PATH: /run/secrets/ca_key
+      CA_CERT_PATH: /run/secrets/ca_cert
     volumes:
       - cache-data:/app/.data
+    secrets:
+      - ca_key
+      - ca_cert
 
 volumes:
   cache-data:
+
+secrets:
+  ca_key:
+    file: ./key.pem
+  ca_cert:
+    file: ./cert.pem
 ```
 
 ### Environment Variables
@@ -32,6 +42,14 @@ volumes:
 
 The base URL of your cache server. This needs to be accessible by your runners as it is used for making API requests and downloading cached files.
 
+#### `CA_KEY_PATH`
+
+Path to the CA key. This is used for proxying HTTPS requests which is needed for intercepting cache requests.
+
+#### `CA_CERT_PATH`
+
+Path to the CA certificate. This is used for proxying HTTPS requests which is needed for intercepting cache requests.
+
 #### `STORAGE_DRIVER`
 
 - Default: `filesystem`
@@ -60,74 +78,67 @@ variant: subtle
 ---
 ::
 
-#### `CLEANUP_OLDER_THAN_DAYS`
+#### `CACHE_CLEANUP_OLDER_THAN_DAYS`
 
 - Default: `90`
 
 The number of days to keep stale cache data and metadata before deleting it. Set to `0` to disable cache cleanup.
 
-#### `NITRO_PORT`
+#### `CACHE_CLEANUP_CRON`
 
-- Default: `3000`
+- Default: `0 0 * * *`
 
-The port the server should listen on.
+The cron schedule for running the cache cleanup job.
 
-## 2. Setup with Self-Hosted Runners
+#### `UPLOAD_CLEANUP_CRON`
 
-To leverage the GitHub Actions Cache Server with your self-hosted runners, you'll need to configure a couple of environment variables on your runners. This ensures that your runners can authenticate with and utilize the cache server effectively.
+- Default: `*/10 * * * *`
 
-### Configuring Environment Variables on Self-Hosted Runners
+The cron schedule for running the upload cleanup job. This job will delete any dangling (failed or incomplete) uploads.
 
-**`ACTIONS_RESULTS_URL`**: This tells your self-hosted runner where to send cache requests. Set this environment variable to the `API_BASE_URL` of your cache server, making sure to include a trailing slash.
+#### `PROXY_PORT`
 
-For example, if your cache server's `API_BASE_URL` is `http://localhost:3000`, you would set `ACTIONS_RESULTS_URL` to `http://localhost:3000/`.
+- Default: `8000`
 
-::u-alert
----
-icon: 'tabler:alert-triangle'
-class: ring-amber-400
-color: amber
-description: Make sure to add a trailing slash to the ACTIONS_RESULTS_URL environment variable.
-variant: subtle
----
-::
+The port the proxy server should listen on.
 
-### Getting the Actions Runner to Use the Cache Server
+#### `NITRO_PORT`
 
-The default self-hosted runner overwrites the `ACTIONS_RESULTS_URL` environment variable with the GitHub-hosted cache server URL. To get the runner to use your self-hosted cache server, you'll need to modify the runner binary:
+- Default: `3000`
 
-#### Docker
+The port the server should listen on.
 
-Just add the following lines to your Dockerfile:
+#### `TEMP_DIR`
 
-```dockerfile [Dockerfile]
-FROM ghcr.io/actions/actions-runner:latest
+- Default: os temp dir
 
-# modify actions runner binaries to allow custom cache server implementation
-RUN sed -i 's/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x55\x00\x52\x00\x4C\x00/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x4F\x00\x52\x00\x4C\x00/g' /home/runner/bin/Runner.Worker.dll
-```
+The directory to use for temporary files.
 
-#### Bare Metal
+## 2. Setup with Self-Hosted Runners
 
-::code-group
+### Generate CA Key and Certificate
 
-```bash [Linux]
-sed -i 's/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x55\x00\x52\x00\x4C\x00/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x4F\x00\x52\x00\x4C\x00/g' /path_to_your_runner/bin/Runner.Worker.dll
+```bash
+openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 3650 -nodes
 ```
 
-```bash [MacOS]
-gsed -i 's/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x55\x00\x52\x00\x4C\x00/\x41\x00\x43\x00\x54\x00\x49\x00\x4F\x00\x4E\x00\x53\x00\x5F\x00\x52\x00\x45\x00\x53\x00\x55\x00\x4C\x00\x54\x00\x53\x00\x5F\x00\x4F\x00\x52\x00\x4C\x00/g' /path_to_your_runner/bin/Runner.Worker.dll
-```
+This will create a `key.pem` and `cert.pem` file in the current directory. These files need to be mounted into the cache server container.
 
-```bash [Windows]
-[byte[]] -split (((Get-Content -Path ./bin/Runner.Worker.dll -Encoding Byte) | ForEach-Object ToString X2) -join '' -Replace '41004300540049004F004E0053005F0052004500530055004C00540053005F00550052004C00','41004300540049004F004E0053005F0052004500530055004C00540053005F004F0052004C00' -Replace '..', '0x$& ') | Set-Content -Path /path_to_your_runner/bin/Runner.Worker.dll -Encoding Byte
-```
+### Update the Dockerfile of your runner image
 
-::
+```dockerfile [Dockerfile]
+# Add the CA certificate to trusted certificates
+RUN sudo apt-get install -y ca-certificates
+RUN echo "<YOUR GENERATED CERTIFICATE>" | sudo tee /usr/local/share/ca-certificates/cache-server-ca.crt
+RUN sudo update-ca-certificates
 
-This will replace the strings `ACTIONS_RESULTS_URL` with `ACTIONS_RESULTS_ORL` in the runner binary. This will prevent the runner from overwriting the `ACTIONS_RESULTS_URL` environment variable and allow it to use your self-hosted cache server.
+# Configure NodeJS to use the CA certificate
+ENV NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/cache-server-ca.crt
 
-Doing it this way is a bit of a hack, but it's easier than compiling your own runner binary from source and works great.
+# Configure proxy
+ENV http_proxy=http://<your cache server>:<PROXY_PORT>
+ENV https_proxy=http://<your cache server>:<PROXY_PORT>
+```
 
 ## 3. Using the Cache Server
 

docs/content/1.getting-started/1.index.md

@@ -1,50 +1,6 @@
 ---
 title: How it works
-description: Learn how we built the GitHub Actions Cache Server and how it works without any workflow file changes
+description: ''
 ---
 
-## 1. Reverse-Engineering the cache server
-
-This is actually a pretty simple process. We just need to look at the requests that the GitHub Actions runner makes to the cache server and then replicate the api routes. We can use the source code of the official `actions/cache` action to see how the requests are made.
-
-GitHub also has very good [documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key) on how the cache keys are matched.
-
-## 2. Getting the actions runner to use our cache server
-
-The whole idea of creating a self-hosted cache server originated from the discovery that the official `actions/cache` action uses an environment variable `ACTIONS_RESULTS_URL` to determine where to send cache requests. This means that we can simply set this environment variable to the base URL of our cache server and the runner will start using it ...right?
-
-Well, not exactly. The `actions/cache` action uses the `ACTIONS_RESULTS_URL` environment variable to determine the base URL of the cache server but we cannot overrid this environment variable in any way.
-
-The default actions runner always overrides the `ACTIONS_RESULTS_URL` environment variable with an internal URL that points to the official GitHub cache server. This is the code that does it:
-
-```c#
-var systemConnection = ExecutionContext.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
-Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri;
-Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
-if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl))
-{
-    Environment["ACTIONS_CACHE_URL"] = cacheUrl;
-}
-if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
-{
-    Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
-}
-if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
-{
-    Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;
-    Environment["ACTIONS_ID_TOKEN_REQUEST_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
-}
-if (systemConnection.Data.TryGetValue("ResultsServiceUrl", out var resultsUrl) && !string.IsNullOrEmpty(resultsUrl))
-{
-    Environment["ACTIONS_RESULTS_URL"] = resultsUrl;
-}
-
-if (ExecutionContext.Global.Variables.GetBoolean("actions_uses_cache_service_v2") ?? false)
-{
-    Environment["ACTIONS_CACHE_SERVICE_V2"] = bool.TrueString;
-}
-```
-
-The line `Environment["ACTIONS_RESULTS_URL"] = resultsUrl;` is the one that overrides the `ACTIONS_RESULTS_URL` environment variable with the internal URL.
-
-To allow overriding the `ACTIONS_RESULTS_URL` environment variable, we need to modify the runner binary. This is a bit tricky because the runner is a compiled binary and we cannot simply modify the source code and recompile it. We need to modify the binary itself. So we did just that. We replaced the string `ACTIONS_RESULTS_URL` with `ACTIONS_RESULTS_ORL` (being careful to keep the same length) in the runner binary. This way, the runner will not override the `ACTIONS_RESULTS_URL` environment variable and we can set it to our cache server's base URL.
+The cache server acts as a proxy for the actions runner. We forward all cache related requests to our cache server implementation and passthrough any non-cache related requests to the original server.

docs/content/1.getting-started/4.how-it-works.md

@@ -2,9 +2,9 @@ title: GitHub Actions Cache Server
 description: Easily deploy your own GitHub actions cache without needing to change any workflow files!
 navigation: false
 hero:
-  title: Self-Hosted Cache Server for GitHub Actions
+  title: Self-Hosted Cache Server<br>for GitHub Actions
   description: Easily deploy your own GitHub actions cache without needing to change any workflow files!
-  orientation: horizontal
+  orientation: vertical
   links:
     - label: Get started
       icon: i-heroicons-arrow-right-20-solid
@@ -13,20 +13,30 @@ hero:
       size: lg
   code: |
     ```yaml [docker-compose.yml]
-    version: '3.9'
-
     services:
       cache-server:
         image: ghcr.io/falcondev-oss/github-actions-cache-server:latest
         ports:
           - '3000:3000'
+          - '8000:8000'
         environment:
           API_BASE_URL: http://localhost:3000
+          CA_KEY_PATH: /run/secrets/ca_key
+          CA_CERT_PATH: /run/secrets/ca_cert
         volumes:
           - cache-data:/app/.data
+        secrets:
+          - ca_key
+          - ca_cert
 
     volumes:
       cache-data:
+
+    secrets:
+      ca_key:
+        file: ./key.pem
+      ca_cert:
+        file: ./cert.pem
     ```
 features:
   items:

docs/content/index.yml

@@ -1,14 +1,24 @@
+import { tmpdir } from 'node:os'
 import { z } from 'zod'
 
 const booleanSchema = z.string().transform((v) => v.toLowerCase() === 'true')
+const portSchema = z.coerce.number().int().min(1).max(65_535)
 
 const envSchema = z.object({
   ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'),
-  CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90),
+  CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90),
+  CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'),
+  UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'),
   API_BASE_URL: z.string().url(),
   STORAGE_DRIVER: z.string().toLowerCase().default('filesystem'),
   DB_DRIVER: z.string().toLowerCase().default('sqlite'),
   DEBUG: booleanSchema.default('false'),
+  PROXY_PORT: portSchema.default(8000),
+  NITRO_PORT: portSchema.default(3000),
+  CA_KEY_PATH: z.string(),
+  CA_CERT_PATH: z.string(),
+  TEMP_DIR: z.string().default(tmpdir()),
+  DISABLE_PROXY: booleanSchema.default('false'),
 })
 
 const parsedEnv = envSchema.safeParse(process.env)