Systematically crawl all websites (#32) Solves #19

* fix: use www for edt website to fix redirect

* feat: cloud service to regularly scrape content

* feat: add action to automatically update cloud using ssh

* chore: update cloud docs

* chore: smarter scripts for crawling and moving crawls around

* fix: working go-ssb-room image

* fix: added missing env var to cloud docker-compose

Author: luandro

.github/workflows/cloud-push-main.yml

@@ -0,0 +1,29 @@
+name: Demo Build & Push
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/cloud/**'
+      - 'scripts/**'
+      - 'services/**'
+
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v2
+      - name: Deploy Stack
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.SSH_CLOUD_HOST }}
+          username: ${{ secrets.SSH_CLOUD_USERNAME }}
+          key: ${{ secrets.SSH_CLOUD_SECRET }}
+          port: ${{ secrets.SSH_CLOUD_PORT }}
+          script: |
+            cd ${{ secrets.CLOUD_FOLDER_PATH }}
+            git checkout .
+            git pull origin main
+            docker compose -f docker/cloud/docker-compose.yml down
+            docker compose -f docker/cloud/docker-compose.yml up -d --build

docker/cloud/.env.example

@@ -1,6 +1,11 @@
 # General
 DIR=~/data
 EMAIL=[email protected]
+PGID=1000
+PUID=1000
+# CRAWL
+CRAWL_TIMER=86400 # 86400 = 1 day
+DOMAIN_CRAWLER=crawl.earthdefenderstoolkit.com
 # F-DROID
 DOMAIN_FDROID=repo.earthdefenderstoolkit.com
 # FILE BROWSER

docker/cloud/docker-compose.yml

@@ -71,9 +71,9 @@ services:
       PGID: 1000
       PUID: 1000
     command: -- c
-  # https://hub.docker.com/r/ksinica/go-ssb-room
+  # https://hub.docker.com/r/cooldracula/go-ssb-room
   ssb-room:
-    image: ksinica/go-ssb-room
+    image: cooldracula/go-ssb-room
     restart: always
     ports:
       - "0.0.0.0:8007:8008" # This is the port SSB clients connect to
@@ -138,7 +138,15 @@ services:
       - 21027:21027/udp
   crawler:
     build: ../../services/crawler
+    restart: always
     volumes:
         - ${DIR}/app-data/crawls:/crawls/
+        - ${DIR}/content:/app/content
+    environment:
+      VIRTUAL_HOST: ${DOMAIN_CRAWLER}
+      LETSENCRYPT_HOST: ${DOMAIN_CRAWLER}
+      LETSENCRYPT_EMAIL: ${EMAIL}
+      VIRTUAL_PORT: 8080
+      CRAWL_TIMER: ${CRAWL_TIMER}
   # TODO: Terrastories, Portal
   # V2: Open Balena

docs/SETUP_SERVER.md renamed to docs/SETUP_CLOUD_SERVER.md

@@ -1,4 +1,4 @@
-# Setting up your own Earth Defenders Toolkit Server
+# Setting up your own Earth Defenders Toolkit Cloud
 
 1. **A cloud provider or a computer**: we recommend [Digital Ocean](https://digitalocean.com) or any computer your organization can provide
 2. **Docker and docker-compose**: Some cloud providers (such as Digital Ocean) have a marketplace with [single-click Docker deployment](https://cloud.digitalocean.com/droplets/new?onboarding_origin=marketplace&appId=87786318&image=docker-20-04&activation_redirect=%2Fdroplets%2Fnew%3FappId%3D87786318%26image%3Ddocker-20-04). You can also install Docker and docker-compose on your machine using a single command:
@@ -18,4 +18,10 @@ curl -fsSL https://raw.githubusercontent.com/jinweijie/install-docker-and-compos
 1. Setup a stronger password for FileBrowser and Syncthing apps
 2. Change the description and moderation strategy for the Secure Scuttlebutt Room app
 3. Using the FileBrowser application create folders for your content
-4. Using the Syncthing application share the content folders
+4. Using the Syncthing application share the content folders
+
+## Actions
+
+To automate updates to the cloud you can fork the official repository and add your own Github Action secrets.
+
+Check the [ssh-action](https://github.com/appleboy/ssh-action) repository to understand the different variables.

services/crawler/Dockerfile

@@ -1,2 +1,5 @@
 FROM webrecorder/browsertrix-crawler
 COPY ./crawl-config.yml /app/crawl-config.yml
+COPY ./crawl.sh /app/crawl.sh
+COPY ./start.sh /app/start.sh
+ENTRYPOINT ["sh", "/app/start.sh"]

services/crawler/crawl-config.yml

@@ -1,13 +1,8 @@
 seeds:
-  - url: https://earthdefenderstoolkit.com
-    include: earthdefenderstoolkit.com/(?:\?lang=)?es
-    depth: 2
+  - url: https://www.earthdefenderstoolkit.com
   - url: https://docs.mapeo.app/
-    depth: 1
   - url: https://docs.terrastories.app/
-    depth: 1
   - url: https://docs.earthdefenderstoolkit.com/
-    depth: 1
 combineWARCs: true
 blockRules:
    - url: googleanalytics.com

services/crawler/crawl.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+echo "Crawling content!"
+crawl --config /app/crawl-config.yml
+echo "Crawling done"
+count=$(ls -1 /crawls/collections/crawl-*/archive/*.warc.gz 2>/dev/null | wc -l)
+if [ $count != 0 ]
+then
+    echo "Creating folders if don't exist"
+    mkdir -p /app/content/old-websites
+    mkdir -p /app/content/offline-websites/
+    echo "Moving old content"
+    mv /app/content/offline-websites/* /app/content/old-websites/
+    echo "Copying content to content folder"
+    mv /crawls/collections/crawl-*/archive/* /app/content/offline-websites/
+    echo "Moved!"
+fi
+echo "Sleeping for $CRAWL_TIMER seconds..."

services/crawler/start.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "Starting Earth Defenders Toolkit crawler"
+# TODO: also run pywb to show what's latest crawl
+while true; do /app/crawl.sh; sleep "$CRAWL_TIMER"; done