Don't listen on :8448 for the Complement federation server

.github/workflows/ci.yaml

@@ -31,18 +31,26 @@ jobs:
             tags: msc2836 dendrite_blacklist
             default_branch: master
 
-    container:
-      image: matrixdotorg/complement  # dockerfiles/ComplementCIBuildkite.Dockerfile
-      env:
-        CI: true
-        DOCKER_BUILDKIT: 1
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2 # Checkout complement
+
+      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          echo "~/go/bin" >> $GITHUB_PATH
+
+      # Similar steps as dockerfiles/ComplementCIBuildkite.Dockerfile but on the host. We need
+      # to do this so we can _be_ the host when running Complement so we can snaffle all the ports. If
+      # we run Complement _in_ Docker then we can't -p all high numbered ports which then breaks federation
+      # servers which listen on random high numbered ports.
+      - name: "Install Complement Dependencies"
+        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
+        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 
       - name: "Checkout corresponding ${{ matrix.homeserver }} branch"
         # This is only done for Synapse since Dendrite's docker file pulls in
@@ -80,9 +88,15 @@ jobs:
         # built docker image).
         if: ${{ matrix.homeserver == 'Synapse' }}
         working-directory: homeserver
+        env:
+          DOCKER_BUILDKIT: 1
 
       - run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/
-      - run: set -o pipefail && go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
+      - run: |
+          set -o pipefail &&
+          go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
         shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
+        name: Run Complement Tests
         env:
           COMPLEMENT_BASE_IMAGE: homeserver
+          DOCKER_BUILDKIT: 1

internal/docker/builder.go

@@ -17,7 +17,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
 	"strings"
 	"time"
 
@@ -39,15 +38,6 @@ var (
 	HostnameRunningDocker = "localhost"
 )
 
-func init() {
-	if os.Getenv("CI") == "true" {
-		log.Println("Running under CI: redirecting localhost to docker host on 172.17.0.1")
-		// this assumes we are running inside docker so they have
-		// forwarded the docker socket to us and we're in a container.
-		HostnameRunningDocker = "172.17.0.1"
-	}
-}
-
 const complementLabel = "complement_context"
 
 type Builder struct {

internal/docker/volumes.go

@@ -1,13 +1,9 @@
 package docker
 
 import (
-	"bufio"
 	"context"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
-	"strings"
 
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
@@ -31,50 +27,21 @@ type VolumeCA struct {
 // Prepare the Certificate Authority volume. This is independent of the homeserver calling Prepare
 // hence the contextual string is unused.
 func (v *VolumeCA) Prepare(ctx context.Context, docker *client.Client, x string) error {
-	// TODO: wrap in a lockfile
-	if os.Getenv("CI") == "true" {
-		// When in CI, Complement itself is a container with the CA volume mounted at /ca.
-		// We need to mount this volume to all homeserver containers to synchronize the CA cert.
-		// This is needed to establish trust among all containers.
-
-		containerID := getContainerID()
-		if containerID == "" {
-			return fmt.Errorf("failed to get container ID")
-		}
-		container, err := docker.ContainerInspect(ctx, containerID)
-		if err != nil {
-			return err
-		}
-		// Get the volume that matches the destination in our complement container
-		for i := range container.Mounts {
-			if container.Mounts[i].Destination == "/ca" {
-				v.source = container.Mounts[i].Name
-				v.typ = container.Mounts[i].Type
-				break
-			}
-		}
-		if v.source == "" {
-			// We did not find a volume. This container might be created without a volume,
-			// or CI=true is passed but we are not running in a container.
-			return fmt.Errorf("CI=true but there is no /ca mounted to Complement's container")
-		}
-	} else {
-		// When not in CI, our CA cert is placed in the current working dir.
-		// We bind mount this directory to all homeserver containers.
-		cwd, err := os.Getwd()
+	// Our CA cert is placed in the current working dir.
+	// We bind mount this directory to all homeserver containers.
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	caCertificateDirHost := path.Join(cwd, "ca")
+	if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
+		err = os.Mkdir(caCertificateDirHost, 0770)
 		if err != nil {
 			return err
 		}
-		caCertificateDirHost := path.Join(cwd, "ca")
-		if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
-			err = os.Mkdir(caCertificateDirHost, 0770)
-			if err != nil {
-				return err
-			}
-		}
-		v.source = path.Join(cwd, "ca")
-		v.typ = mount.TypeBind
 	}
+	v.source = path.Join(cwd, "ca")
+	v.typ = mount.TypeBind
 	return nil
 }
 
@@ -108,60 +75,3 @@ func (v *VolumeAppService) Mount() mount.Mount {
 		Target: "/appservices",
 	}
 }
-
-func getContainerID() string {
-	cid, err := getContainerIDViaCPUSet()
-	if err == nil {
-		return cid
-	}
-	fmt.Printf("failed to get container ID via cpuset, trying alternatives: %s\n", err)
-
-	cid, err = getContainerIDViaCGroups()
-	if err == nil {
-		return cid
-	}
-
-	fmt.Printf("failed to get container ID via cgroups, out of options: %s\n", err)
-	return ""
-}
-
-func getContainerIDViaCGroups() (string, error) {
-	file, err := os.Open("/proc/self/cgroup")
-	if err != nil {
-		return "", err
-	}
-
-	scanner := bufio.NewScanner(file)
-	defer file.Close()
-
-	scanner.Split(bufio.ScanLines)
-	for scanner.Scan() {
-		// Returns entries like this on github actions
-		// 9:memory:/actions_job/c8d555525bad6cd896c5aa985ef68010be47b1fb321c95547761c8f1a053b86e
-		line := scanner.Text()
-		segments := strings.Split(line, "/")
-		containerID := segments[len(segments)-1]
-		if containerID == "" || len(containerID) < 64 {
-			continue
-		}
-		return containerID, nil
-	}
-	return "", fmt.Errorf("faild to find container id in cgroups")
-}
-
-func getContainerIDViaCPUSet() (string, error) {
-	// /proc/1/cpuset should be /docker/<containerID>
-	cpuset, err := ioutil.ReadFile("/proc/1/cpuset")
-	if err != nil {
-		return "", err
-	}
-	if !strings.Contains(string(cpuset), "docker") {
-		return "", fmt.Errorf("could not identify container ID using /proc/1/cpuset - cpuset=%s", string(cpuset))
-	}
-	cpusetList := strings.Split(strings.TrimSpace(string(cpuset)), "/")
-	containerID := cpusetList[len(cpusetList)-1]
-	if len(containerID) == 0 {
-		return "", fmt.Errorf("cpuset missing container ID")
-	}
-	return containerID, nil
-}

internal/federation/handle.go

@@ -18,7 +18,7 @@ import (
 func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
 	// Check federation signature
 	fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-		req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+		req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 	)
 	if fedReq == nil {
 		w.WriteHeader(errResp.Code)
@@ -74,7 +74,7 @@ func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
 // HandleMakeSendJoinRequests.
 func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
 	fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-		req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+		req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 	)
 	if fedReq == nil {
 		w.WriteHeader(errResp.Code)
@@ -106,9 +106,9 @@ func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
 
 	// return state and auth chain
 	b, err := json.Marshal(gomatrixserverlib.RespSendJoin{
+		Origin:      gomatrixserverlib.ServerName(s.serverName),
 		AuthEvents:  authEvents,
 		StateEvents: stateEvents,
-		Origin:      gomatrixserverlib.ServerName(s.ServerName),
 	})
 	if err != nil {
 		w.WriteHeader(500)
@@ -141,7 +141,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
 		// https://matrix.org/docs/spec/server_server/r0.1.4#put-matrix-federation-v2-invite-roomid-eventid
 		s.mux.Handle("/_matrix/federation/v2/invite/{roomID}/{eventID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-				req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
+				req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
 			)
 			if fedReq == nil {
 				w.WriteHeader(errResp.Code)
@@ -169,7 +169,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
 			}
 
 			// Sign the event before we send it back
-			signedEvent := inviteRequest.Event().Sign(s.ServerName, s.KeyID, s.Priv)
+			signedEvent := inviteRequest.Event().Sign(s.serverName, s.KeyID, s.Priv)
 
 			// Send the response
 			res := map[string]interface{}{
@@ -195,7 +195,7 @@ func HandleDirectoryLookups() func(*Server) {
 				b, err := json.Marshal(gomatrixserverlib.RespDirectory{
 					RoomID: roomID,
 					Servers: []gomatrixserverlib.ServerName{
-						gomatrixserverlib.ServerName(s.ServerName),
+						gomatrixserverlib.ServerName(s.serverName),
 					},
 				})
 				if err != nil {
@@ -235,7 +235,7 @@ func HandleEventRequests() func(*Server) {
 			}
 
 			txn := gomatrixserverlib.Transaction{
-				Origin:         gomatrixserverlib.ServerName(srv.ServerName),
+				Origin:         gomatrixserverlib.ServerName(srv.serverName),
 				OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
 				PDUs: []json.RawMessage{
 					event.JSON(),
@@ -259,7 +259,7 @@ func HandleKeyRequests() func(*Server) {
 		keymux := srv.mux.PathPrefix("/_matrix/key/v2").Subrouter()
 		keyFn := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			k := gomatrixserverlib.ServerKeys{}
-			k.ServerName = gomatrixserverlib.ServerName(srv.ServerName)
+			k.ServerName = gomatrixserverlib.ServerName(srv.serverName)
 			publicKey := srv.Priv.Public().(ed25519.PublicKey)
 			k.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
 				srv.KeyID: {
@@ -276,7 +276,7 @@ func HandleKeyRequests() func(*Server) {
 			}
 
 			k.Raw, err = gomatrixserverlib.SignJSON(
-				string(srv.ServerName), srv.KeyID, srv.Priv, toSign,
+				string(srv.serverName), srv.KeyID, srv.Priv, toSign,
 			)
 			if err != nil {
 				w.WriteHeader(500)
@@ -304,9 +304,9 @@ func HandleMediaRequests(mediaIds map[string]func(w http.ResponseWriter)) func(*
 			origin := vars["origin"]
 			mediaId := vars["mediaId"]
 
-			if origin != srv.ServerName {
+			if origin != srv.serverName {
 				w.WriteHeader(400)
-				w.Write([]byte("complement: Invalid Origin; Expected " + srv.ServerName))
+				w.Write([]byte("complement: Invalid Origin; Expected " + srv.serverName))
 				return
 			}
 
@@ -338,7 +338,7 @@ func HandleTransactionRequests(pduCallback func(*gomatrixserverlib.Event), eduCa
 
 			// Check federation signature
 			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
-				req, time.Now(), gomatrixserverlib.ServerName(srv.ServerName), srv.keyRing,
+				req, time.Now(), gomatrixserverlib.ServerName(srv.serverName), srv.keyRing,
 			)
 			if fedReq == nil {
 				log.Printf(