Switch to split known server lists

See raiden-network/raiden#6442

Author: ulope

build/purger/purger.py

@@ -125,7 +125,18 @@ def purge(
                 # In case an empty env var is set
                 url_known_federation_servers = DEFAULT_MATRIX_KNOWN_SERVERS[Environment.PRODUCTION]
             # fetch remote whiltelist
-            remote_whitelist = yaml.safe_load(requests.get(url_known_federation_servers).text)
+            try:
+                remote_whitelist = json.loads(requests.get(url_known_federation_servers).text)[
+                    "all_servers"
+                ]
+            except (requests.RequestException, JSONDecodeError, KeyError) as ex:
+                click.secho(
+                    f"Error while fetching whitelist: {ex!r}. "
+                    f"Ignoring, containers will be restarted.",
+                    err=True,
+                )
+                # An empty whitelist will cause the container to be restarted
+                remote_whitelist = []
 
             client = docker.from_env()  # pylint: disable=no-member
             for container in client.containers.list():
@@ -145,16 +156,9 @@ def purge(
                         continue
 
                     click.secho(f"Whitelist changed. Restarting. new_list={remote_whitelist!r}")
-                except (
-                    KeyError,
-                    IndexError,
-                    requests.RequestException,
-                    yaml.scanner.ScannerError,
-                ) as ex:
+                except (KeyError, IndexError) as ex:
                     click.secho(
-                        f"An error ocurred while fetching whitelists: {ex!r}\n"
-                        "Restarting anyway",
-                        err=True,
+                        f"Error fetching container status: {ex!r}. Restarting anyway.", err=True,
                     )
                 # restart container
                 container.restart(timeout=30)

build/room_ensurer/room_ensurer.py

@@ -26,8 +26,6 @@
 
 patch_all()  # isort:skip
 
-from raiden.utils.datastructures import merge_dict
-
 import json
 import os
 import sys
@@ -42,7 +40,6 @@
 import gevent
 from eth_utils import encode_hex, to_normalized_address
 from matrix_client.errors import MatrixError
-from raiden_contracts.utils.type_aliases import ChainID
 from structlog import get_logger
 
 from raiden.constants import (
@@ -51,14 +48,16 @@
     PATH_FINDING_BROADCASTING_ROOM,
     Environment,
     Networks,
+    ServerListType,
 )
 from raiden.log_config import configure_logging
 from raiden.network.transport.matrix import make_room_alias
 from raiden.network.transport.matrix.client import GMatrixHttpApi
 from raiden.settings import DEFAULT_MATRIX_KNOWN_SERVERS
 from raiden.tests.utils.factories import make_signer
 from raiden.utils.cli import get_matrix_servers
-
+from raiden.utils.datastructures import merge_dict
+from raiden_contracts.utils.type_aliases import ChainID
 
 ENV_KEY_KNOWN_SERVERS = "URL_KNOWN_FEDERATION_SERVERS"
 
@@ -104,7 +103,9 @@ def __init__(
 
         self._known_servers: Dict[str, str] = {
             urlparse(server_url).netloc: server_url
-            for server_url in get_matrix_servers(known_servers_url)
+            for server_url in get_matrix_servers(
+                known_servers_url, server_list_type=ServerListType.ALL_SERVERS
+            )
         }
         if not self._known_servers:
             raise RuntimeError(f"No known servers found from list at {known_servers_url}.")
@@ -214,7 +215,7 @@ def _ensure_room_for_network(self, network: Networks, alias_fragment: str) -> No
         self._ensure_admin_power_levels(room_infos[self._own_server_name])
 
     def _join_and_alias_room(
-            self, first_server_room_alias: str, own_server_room_alias: str
+        self, first_server_room_alias: str, own_server_room_alias: str
     ) -> None:
         response = self._own_api.join_room(first_server_room_alias)
         own_room_id = response.get("room_id")
@@ -291,12 +292,16 @@ def _ensure_admin_power_levels(self, room_info: RoomInfo) -> None:
             return
 
         if own_user not in current_power_levels["users"]:
-            log.warning(f"{own_user} has not been granted administrative power levels yet. Doing nothing.")
+            log.warning(
+                f"{own_user} has not been granted administrative power levels yet. Doing nothing."
+            )
             return
 
         # the supposed power level dict could be just a subset of the current
         # because providers who left cannot be removed from other admins
-        if set(supposed_power_levels["users"].keys()).issubset(set(current_power_levels["users"].keys())):
+        if set(supposed_power_levels["users"].keys()).issubset(
+            set(current_power_levels["users"].keys())
+        ):
             log.debug(f"Power levels are up to date. Doing nothing.")
             return
 

build/synapse/render_config_template.py

@@ -2,16 +2,23 @@
 import os
 import random
 import string
+from json.decoder import JSONDecodeError
 from pathlib import Path
 from typing import Optional
 from urllib.error import URLError
 from urllib.request import urlopen
 
-PATH_CONFIG = Path("/config/synapse.yaml")
-PATH_CONFIG_TEMPLATE = Path("/config/synapse.template.yaml")
+import click
+from docker import Client
+
+PATH_CONFIG_SYNAPSE = Path("/config/synapse.yaml")
+PATH_CONFIG_TEMPLATE_SYNAPSE = Path("/config/synapse.template.yaml")
+PATH_CONFIG_WORKER_BASE = Path("/config/workers/")
+PATH_CONFIG_TEMPLATE_WORKER = Path("/config/worker.template.yaml")
+
 PATH_MACAROON_KEY = Path("/data/keys/macaroon.key")
 PATH_ADMIN_USER_CREDENTIALS = Path("/config/admin_user_cred.json")
-PATH_KNOWN_FEDERATION_SERVERS = Path("/data/known_federation_servers.yaml")
+PATH_KNOWN_FEDERATION_SERVERS = Path("/data/known_federation_servers.json")
 PATH_WELL_KNOWN_FILE = Path("/data_well_known/server")
 
 # This file gets created during docker build from the given Raiden version
@@ -36,7 +43,13 @@ def get_known_federation_servers(url_known_federation_servers: Optional[str]) ->
     try:
         resp = urlopen(url_known_federation_servers)
         if 200 <= resp.code < 300:
-            PATH_KNOWN_FEDERATION_SERVERS.write_text(resp.read().decode())
+            try:
+                known_servers = json.loads(resp.read().decode())
+                PATH_KNOWN_FEDERATION_SERVERS.write_text(
+                    "".join(f"- {server}\n" for server in known_servers["all_servers"])
+                )
+            except (JSONDecodeError, KeyError):
+                print("Error loading known servers list:", resp.code, resp.read().decode())
         else:
             print("Error fetching known servers list:", resp.code, resp.read().decode())
     except URLError as ex:
@@ -47,13 +60,13 @@ def get_known_federation_servers(url_known_federation_servers: Optional[str]) ->
 
 
 def render_synapse_config(server_name: str, url_known_federation_servers: Optional[str]) -> None:
-    template_content = PATH_CONFIG_TEMPLATE.read_text()
+    template_content = PATH_CONFIG_TEMPLATE_SYNAPSE.read_text()
     rendered_config = string.Template(template_content).substitute(
         MACAROON_KEY=get_macaroon_key(),
         SERVER_NAME=server_name,
         KNOWN_SERVERS=get_known_federation_servers(url_known_federation_servers),
     )
-    PATH_CONFIG.write_text(rendered_config)
+    PATH_CONFIG_SYNAPSE.write_text(rendered_config)
 
 
 def render_well_known_file(server_name: str) -> None:

known_servers/known_servers-development-v1.2.0.json

@@ -0,0 +1,14 @@
+{
+    "active_servers": [
+        "transport.transport01.raiden.network",
+        "transport.transport02.raiden.network",
+        "transport.transport03.raiden.network",
+        "transport.transport04.raiden.network"
+    ],
+    "all_servers": [
+        "transport.transport01.raiden.network",
+        "transport.transport02.raiden.network",
+        "transport.transport03.raiden.network",
+        "transport.transport04.raiden.network"
+    ]
+}

known_servers/known_servers-production-v1.2.0.json

@@ -0,0 +1,13 @@
+{
+    "active_servers": [
+        "transport.mainnet.raiden.anyblock.tools",
+        "transport.raiden.dappnode.io",
+        "transport.raiden.badgateway.tech"
+    ],
+    "all_servers": [
+        "transport.mainnet.raiden.anyblock.tools",
+        "transport.raiden.dappnode.io",
+        "transport.raiden.badgateway.tech",
+        "transport.raiden.overdoze.se"
+    ]
+}