Collapse GitHub Actions into single job to save minutes

.github/workflows/ww_conversion_factor_US.yml → .github/workflows/all-countries.yml

@@ -1,10 +1,10 @@
-name: Update US Wastewater data (conversion factor approach)
+name: Update wastewater data for all countries
 
 on:  # yamllint disable-line rule:truthy
   push:
     branches: [main]
   schedule:
-    - cron: '57 09 * * *'
+    - cron: '57 09 */3 * *'
 
 jobs:
   build:
@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
+        pip install flake8 pytest selenium webdriver_manager
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
     - name: Lint with flake8
@@ -34,12 +34,41 @@ jobs:
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-    - name: Process Data
+    - name: Get and process data for all countries
       run: |
-        python ww_factor_US_states.py
-        python ww_factor_US.py
+        # Canada
+        python scripts/canada_get_data.py || true
 
-    - name: commit files
+        # Finland
+        python scripts/finland_get_ww_data.py || true
+
+        # Germany
+        python scripts/ww_Ger.py         \
+        && python scripts/ww_raw_Ger.py  \
+        || true
+
+        # Netherlands
+        python scripts/get_data_Netherlands.py || true
+
+        # Slovenia
+        python scripts/slovenia_get_ww_data.py
+
+        # Sweden
+        swe_csv=https://datagraphics.dckube.scilifelab.se/dataset/0ac8fa02871745048491de74e5689da9.csv
+        curl -L $swe_csv --output "data/Sweden/Sweden_ww.csv" || true
+
+        # Switzerland
+        python scripts/switzerland_get_ww_data.py || true
+
+        # United States (conversion factor approach)
+        python ww_factor_US_states.py && python ww_factor_US.py
+
+        # US CDC
+        python get_virus_concentration_wastewater_cdc.py  \
+        && python get_wastewater_change_cdc.py            \
+        || true
+
+    - name: Commit all updated files
       run: |
         git config --local user.email "[email protected]"
         git config --local user.name "GitHub Action"