Dockerize scraper, upgrade to puppeteer 3.0.0

Author: TomyCesaille

.dockerignore

@@ -5,4 +5,5 @@
 docker-compose.yml
 docker-compose.*.yml
 node_modules
+npm-debug.log
 data/*

Dockerfile

@@ -0,0 +1,6 @@
+FROM schliflo/docker-puppeteer:3.0.0
+WORKDIR /app
+COPY package*.json ./
+RUN npm install
+COPY . .
+CMD ["node", "index.js"] 

README.md

@@ -3,42 +3,42 @@
 Proof of concept about tracking contacts in WhatsApp.
 Check out my [blog entry](https://jorislacance.fr/blog/2020/04/01/whatsapp-tracking) for in-depth info.
 
-![grafana](https://i.imgur.com/MMq8q4u.png)  
+![grafana](https://i.imgur.com/MMq8q4u.png)
 
 ## architecture
 
 The POC is composed of:
 
-* The Node.js WhatsApp scraper robot - Gather the data
-* The InfluxDB 2.0 service - Store the data
-* The Grafana 6.7 service - Visualize the data
+- The Node.js WhatsApp scraper robot - Gather the data
+- The InfluxDB 2.0 service - Store the data
+- The Grafana 6.7 service - Visualize the data
 
 ## setup
 
 ```bash
 # run the InfluxDB and Grafana services
-docker-compose up -d
+docker-compose up -d influxdb grafana
 ```
 
 ### influxDB setup
 
-* Go to [http://localhost:9999](http://localhost:9999), setup an admin account
-  * Name the initial organization like you want, `initial-org` for instance
-  * Name the initial bucket anything, like `yolo`, we won't use the initial one because there will be sample data in it
-* Create a new bucket `whatsapp-tracking`
-* Generate a token `whatsapp-tracking-scraper` with write permission to `whatsapp-tracking` bucket
-* Generate a token `grafana` with 'all access'
+- Go to [http://localhost:9999](http://localhost:9999), setup an admin account
+  - Name the initial organization like you want, `initial-org` for instance
+  - Name the initial bucket anything, like `yolo`, we won't use the initial one because there will be sample data in it
+- Create a new bucket `whatsapp-tracking`
+- Generate a token `whatsapp-tracking-scraper` with write permission to `whatsapp-tracking` bucket
+- Generate a token `grafana` with 'all access'
 
 ### grafana setup
 
-* Go to [http://localhost:3000](http://localhost:3000), setup an admin account
-* Add the data source using the plugin `Flux (InfluxDB) [BETA]` (for InfluxDB 2.0)
-  * URL: `http://influxdb:9999`
-  * with credentials: `true`
-  * Organization: `initial-org`
-  * Default Bucket: `whatsapp-tracking`
-  * Token: the grafana token
-* Import the dashboard (file `grafana-dashboard.json`)
+- Go to [http://localhost:3000](http://localhost:3000), setup an admin account
+- Add the data source using the plugin `Flux (InfluxDB) [BETA]` (for InfluxDB 2.0)
+  - URL: `http://influxdb:9999`
+  - with credentials: `true`
+  - Organization: `initial-org`
+  - Default Bucket: `whatsapp-tracking`
+  - Token: the grafana token
+- Import the dashboard (file `grafana-dashboard.json`)
 
 ### scraper setup
 
@@ -51,19 +51,23 @@ INFLUXDB_ORG=initial-org
 INFLUXDB_BUCKET=whatsapp-tracking
 ```
 
-## scraper usage
+## scraper usage Windows
 
-```bash
+```powershell
 # init the robot
 npm install
 # run the robot
 node index.js
 ```
 
-## limitation
+## scraper usage docker
 
-Not sure if the scraper runs on Linux...
+```bash
+docker-compose up scraper
+# Look at `./data/screenshots/` to get the QR code peering.
+```
 
 ## todo
 
-* Dockerize the scraper too ? Hardly with the peering procedure that requires us to scan the QR code.
+- [x] Dockerize the scraper too ? Hardly with the peering procedure that requires us to scan the QR code. -> screenshot through file system
+- [ ] Track several contacts by rotating the contact tracked every hours or so (we will loose the precise `online` state tracking but gather more `last seen` data)

docker-compose.yml

@@ -15,15 +15,35 @@ services:
     restart: unless-stopped
 
   grafana:
-      image: grafana/grafana:6.7.2
-      ports:
-        - "3000:3000"
-      volumes:
-        - ./data/grafana:/var/lib/grafana
-      environment:
-        - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-influxdb-flux-datasource,natel-discrete-panel,petrslavotinek-carpetplot-panel,vonage-status-panel,flant-statusmap-panel,neocat-cal-heatmap-panel
-      logging:
-        options:
-          max-size: "50m"
-          max-file: "10"
-      restart: unless-stopped
+    image: grafana/grafana:6.7.2
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./data/grafana:/var/lib/grafana
+    environment:
+      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-influxdb-flux-datasource,natel-discrete-panel,petrslavotinek-carpetplot-panel,vonage-status-panel,flant-statusmap-panel,neocat-cal-heatmap-panel
+    logging:
+      options:
+        max-size: "50m"
+        max-file: "10"
+    restart: unless-stopped
+
+  scraper:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    shm_size: "1gb"
+    volumes:
+      - ./data/userdatadocker:/usr/data/userdata
+      - ./data/screenshots:/usr/data/screenshots
+    environment:
+      - CONTACT_TARGET=${CONTACT_TARGET}
+      - INFLUXDB_TOKEN=${INFLUXDB_TOKEN}
+      - INFLUXDB_ORG=${INFLUXDB_ORG}
+      - INFLUXDB_BUCKET=${INFLUXDB_BUCKET}
+      - ISDOCKER=TRUE
+    logging:
+      options:
+        max-size: "50m"
+        max-file: "10"
+    restart: unless-stopped

index.js

@@ -1,30 +1,42 @@
 const puppeteer = require('puppeteer');
 const chrono = require('chrono-node');
-const { InfluxDB, FluxTableMetaData } = require('@influxdata/influxdb-client')
+const { InfluxDB } = require('@influxdata/influxdb-client')
 require('dotenv').config()
 console.log(`CONTACT_TARGET=${process.env.CONTACT_TARGET}`);
 console.log(`INFLUXDB_TOKEN=${process.env.INFLUXDB_TOKEN}`);
 console.log(`INFLUXDB_ORG=${process.env.INFLUXDB_ORG}`);
 console.log(`INFLUXDB_BUCKET=${process.env.INFLUXDB_BUCKET}`);
+console.log(`ISDOCKER=${process.env.ISDOCKER}`);
 
 // The contact name to track (mind the case).
 const contactTarget = process.env.CONTACT_TARGET;
 
-let client = new InfluxDB({ url: 'http://localhost:9999', token: process.env.INFLUXDB_TOKEN });
+let docker = process.env.ISDOCKER === 'TRUE';
+let influxUrl = docker ? `http://influxdb:9999` : 'http://localhost:9999';
+let userDataDir = docker ? `/usr/data/userdata` : 'data/userdata';
+let args = docker ? ['--no-sandbox', '--disable-setuid-sandbox'] : [];
+let screenshotPath = docker ? '/usr/data/screenshots/screenshot.png' : 'data/screenshots/screenshot.png';
+
+let client = new InfluxDB({ url: influxUrl, token: process.env.INFLUXDB_TOKEN });
 const writeApi = client.getWriteApi(process.env.INFLUXDB_ORG, process.env.INFLUXDB_BUCKET);
 
 (async () => {
     const browser = await puppeteer.launch({
-        headless: false, // No headless to scan the QR code.
-        userDataDir: 'data/userdata' // Persist the session.
+        args: args,
+        headless: true,
+        userDataDir: userDataDir // Persist the session.
     });
 
     const page = await browser.newPage();
+    page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0');
+
     await page.goto('https://web.whatsapp.com/');
-    await page.waitFor(5000);
+    await page.waitFor(10000);
+
+    await page.screenshot({ path: screenshotPath });
 
     console.log('Awaiting/Checking peering with WhatsApp phone');
-    await page.waitFor('#side', { timeout: 60000 }).then(() => { // Scan the QR code within the next minute.
+    await page.waitFor('#side', { timeout: 120000 }).then(() => { // Scan the QR code within the next 2 minutes.
         console.log('Connected !');
     }).catch((res) => {
         console.log('Not connected !', res);