Dev ops changes for the k8s migrations

rpc endpoints
building cd system for the CI

Author: Acollie

Makefile

@@ -1,7 +1,7 @@
-# Start docker containers
-start:
-	docker-compose up -d
+PROTO_SRC_DIR := protos
+PROTO_OUT_DIR := pkg/generated
 
-# Stop docker containers
-stop:
-	docker-compose down
+buildSpider:
+	docker buildx build --platform linux/amd64 -f cmd/spider/Dockerfile -t spider .
+	docker tag spider:latest 967991486854.dkr.ecr.eu-west-1.amazonaws.com/spider:latest
+	docker push 967991486854.dkr.ecr.eu-west-1.amazonaws.com/spider:latest

README.md

@@ -4,7 +4,6 @@
 ![Example of a graph](/assets/example.png "Example of a graph")
 
 ## Overview 🌐
-![Overview of architecture](/assets/overview.png "Overview")
 
 ### 1. Spider
 
@@ -30,7 +29,12 @@ This front end which allows for the Database to be searched
 ## Design considerations
 
 - Why Not use OS/Elastic search
-- 
+
+## Domain specific language
+
+For this project I have decided to created a domain specfic language for creating indexes and for querying the database.
+Overall this was a good opportunity to learn more about compiler design and to create a tool which would leverage a
+massive amount of flexibility
 
 
 

cmd/spider/Dockerfile

@@ -1,4 +1,28 @@
-FROM go.1.22
-LABEL authors="alexcollie"
-# Go lang installation
-ENTRYPOINT ["top", "-b"]
+FROM golang:latest
+
+# Set working directory
+WORKDIR /app
+
+# Copy go.mod and go.sum
+COPY go.mod go.sum ./
+
+# Download dependencies
+RUN go mod download
+
+# Copy the entire project
+COPY . .
+
+# Build the application
+#RUN go build -o spider cmd/spider/main.go
+RUN GOOS=linux GOARCH=amd64 go build -o spider cmd/spider/main.go
+
+
+# Expose ports
+EXPOSE 9090 9090
+# Used for health check
+EXPOSE 8080 8080
+# Used for metrics
+EXPOSE 80 80
+
+# Run the application
+CMD ["./spider"]

cmd/spider/deployment.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spider-deployment
+  annotations:
+    iam.amazonaws.com/role: ${IAM_ROLE}
+  labels:
+    app: spider
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: spider
+  template:
+    metadata:
+      labels:
+        app: spider
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: /metrics
+        prometheus.io/port: "8080"
+    spec:
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: spider
+          image: ${SPIDER_IMAGE}
+          ports:
+            - name: grpc
+              containerPort: 9090
+            - name: metrics
+              containerPort: 8080
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "128Mi"
+            limits:
+              cpu: "500m"
+              memory: "128Mi"

cmd/spider/handler/flow_test.go

@@ -0,0 +1,43 @@
+package handler
+
+import (
+	"context"
+	"webcrawler/cmd/spider/pkg/db"
+	"webcrawler/pkg/generated/service/spider"
+)
+
+type RpcServer struct {
+	spider.UnimplementedSpiderServer
+
+	db db.Db
+}
+
+func (c *RpcServer) mustEmbedUnimplementedSpiderServer() {
+	//TODO implement me
+	panic("implement me")
+}
+
+func NewRPCServer(db db.Db) *RpcServer {
+	return &RpcServer{
+		db: db,
+	}
+}
+
+func (c *RpcServer) GetSeenList(ctx context.Context, request *spider.SeenListRequest) (*spider.SeenListResponse, error) {
+	pages, err := c.db.Page.GetAllPages(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var response []*spider.Page
+	for _, page := range pages {
+		response = append(response, &spider.Page{
+			Url:   page.Url,
+			Body:  page.Body,
+			Links: page.Links,
+		})
+	}
+
+	return &spider.SeenListResponse{
+		SeenSites: response,
+	}, nil
+}

cmd/spider/handler/rpc.go

@@ -4,11 +4,20 @@ import (
 	"context"
 	"fmt"
 	"github.com/joho/godotenv"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
 	"log"
+	"net"
 	"net/http"
 	"webcrawler/cmd/spider/handler"
 	"webcrawler/cmd/spider/pkg/db"
 	"webcrawler/pkg/config"
+	"webcrawler/pkg/generated/service/spider"
+	"webcrawler/pkg/health"
 )
 
 func main() {
@@ -66,15 +75,35 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	// Initialize OpenTelemetry
+	exporter, err := prometheus.New()
+	if err != nil {
+		log.Fatalf("Failed to create Prometheus exporter: %v", err)
+	}
+	meterProvider := metric.NewMeterProvider(metric.WithReader(exporter))
+	otel.SetMeterProvider(meterProvider)
 
+	// K8s health check
+	http.HandleFunc("/health", health.Ok)
+	http.Handle("/metrics", promhttp.Handler())
+
+	// Start HTTP server in a separate goroutine
+	go func() {
+		log.Fatal(http.ListenAndServe("0.0.0.0:8080", nil))
+	}()
 	go func() {
 		server.Scan(ctx)
 	}()
-	http.HandleFunc("/seen", func(writer http.ResponseWriter, request *http.Request) {
 
-	})
+	// GRPC server
+	var opts []grpc.ServerOption
+	lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", 9090))
+	if err != nil {
+		panic(err)
+	}
+	grpcServer := grpc.NewServer(opts...)
+	spider.RegisterSpiderServer(grpcServer, handler.NewRPCServer(server.Db))
+	reflection.Register(grpcServer)
+	grpcServer.Serve(lis)
 
-	http.HandleFunc("/health", func(writer http.ResponseWriter, request *http.Request) {
-		fmt.Fprintf(writer, "OK")
-	})
 }

cmd/spider/main.go

@@ -0,0 +1,130 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/go-faker/faker/v4"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"webcrawler/cmd/spider/pkg/db/page"
+	"webcrawler/cmd/spider/pkg/db/queue"
+	"webcrawler/cmd/spider/pkg/site"
+	dbx "webcrawler/pkg/db"
+	"webcrawler/pkg/sqlx"
+	test_containers "webcrawler/pkg/test-containers"
+)
+
+type testDb struct {
+	conn *sql.DB
+	name string
+}
+
+func Test_DbInteraction(t *testing.T) {
+	ctx := context.Background()
+	sqliteConn, err := dbx.NewSqlite()
+	require.NoError(t, err)
+	defer sqliteConn.Close()
+	sqliteTestDb := testDb{
+		conn: sqliteConn,
+		name: "sqlite",
+	}
+	defer func() {
+		_, err := sqliteConn.Exec(sqlx.DropSeenPages)
+		require.NoError(t, err)
+		_, err = sqliteConn.Exec(sqlx.DropQueue)
+		require.NoError(t, err)
+	}()
+
+	mariaConn, testContainer, err := test_containers.NewMarina(ctx)
+	require.NoError(t, err)
+	defer testContainer.Terminate(ctx)
+	mariaTestDb := testDb{
+		conn: mariaConn,
+		name: "maria",
+	}
+
+	for _, connType := range []testDb{sqliteTestDb, mariaTestDb} {
+		conn := connType.conn
+		t.Run(fmt.Sprintf("conn name:%s basic testing Seen pages", connType.name), func(t *testing.T) {
+			sitePage := site.Page{
+				Url:   faker.URL(),
+				Title: faker.Sentence(),
+				Body:  faker.Paragraph(),
+			}
+			_, err = conn.Exec(sqlx.CreateSeenTable)
+			require.NoError(t, err)
+
+			_, err = conn.Exec(sqlx.CreateSeenTableIndex)
+			require.NoError(t, err)
+
+			require.NoError(t, err)
+			if conn == nil {
+				t.Errorf("Expected connection to be created")
+			}
+			db := page.Db{Sql: conn}
+			err = db.SavePage(ctx, sitePage)
+			require.NoError(t, err)
+
+			page, err := db.GetPage(ctx, sitePage.Url)
+			require.NoError(t, err)
+			require.Equal(t, &sitePage, page)
+
+			//Page which does not exist
+			page, err = db.GetPage(ctx, faker.URL())
+			require.NoError(t, err)
+			require.Nil(t, page)
+
+			// Update page
+			sitePage.Title = faker.Sentence()
+			sitePage.Body = faker.Paragraph()
+			err = db.UpdatePage(ctx, sitePage)
+			require.NoError(t, err)
+			page, err = db.GetPage(ctx, sitePage.Url)
+			require.NoError(t, err)
+			require.Equal(t, &sitePage, page)
+
+			// Remove a page
+			err = db.DeletePage(ctx, sitePage.Url)
+			require.NoError(t, err)
+			// Check if the page is removed
+			page, err = db.GetPage(ctx, sitePage.Url)
+			require.NoError(t, err)
+		})
+		t.Run("queue testing", func(t *testing.T) {
+
+			_, err = conn.Exec(sqlx.CreateQueueTable)
+			require.NoError(t, err)
+
+			db := queue.Db{Sql: conn}
+
+			err = db.AddLink(ctx, "https://example.com")
+			require.NoError(t, err)
+			err = db.AddLink(ctx, "https://example.com/2")
+			require.NoError(t, err)
+
+			links, err := db.GetExplore(ctx)
+			require.NoError(t, err)
+			require.Len(t, links, 2)
+			require.Equal(t, "https://example.com", links[0])
+			require.Equal(t, "https://example.com/2", links[1])
+			err = db.RemoveLink(ctx, "https://example.com")
+			require.NoError(t, err)
+			links, err = db.GetExplore(ctx)
+			require.NoError(t, err)
+			require.Len(t, links, 1)
+			require.Equal(t, "https://example.com/2", links[0])
+
+			err = db.RemoveLink(ctx, "https://example.com/2")
+			require.NoError(t, err)
+			links, err = db.GetExplore(ctx)
+			require.NoError(t, err)
+			require.Len(t, links, 0)
+
+			links = []string{"https://example.com", "https://example.com/2", "https://example.com/3"}
+			err = db.AddLinks(ctx, links)
+			require.NoError(t, err)
+			links, err = db.GetExplore(ctx)
+		})
+	}
+}