Merge pull request #515 from impresso/develop

Release v3.1.0

Author: theorm

.mocharc-integration.json

@@ -1,9 +1,13 @@
 {
+  "$schema": "https://json.schemastore.org/mocharc.json",
+  "require": "tsx",
   "diff": true,
   "package": "./package.json",
   "reporter": "spec",
   "ui": "bdd",
+  "exit": true,
   "recursive": true,
-  "watch-files": ["src/**/*.js", "test/**/*.js"],
+  "extension": ["ts", "js"],
+  "watch-files": ["src/**/*.{js,ts}", "test/**/*.{js,ts}"],
   "ignore": ["node_modules"]
 }

.mocharc.json

@@ -1,12 +1,13 @@
 {
-  "require": "ts-node/register",
-  "loader": "ts-node/esm",
-  "extensions": ["ts", "tsx"],
+  "$schema": "https://json.schemastore.org/mocharc.json",
+  "require": "tsx",
   "diff": true,
   "package": "./package.json",
   "reporter": "spec",
   "ui": "bdd",
+  "exit": true,
   "recursive": true,
-  "watch-files": ["src/**/*.js", "test/**/*.js"],
-  "ignore": ["node_modules", "test/integration/*.js", "test/integration/services/*.js"]
+  "extension": ["ts", "js"],
+  "watch-files": ["src/**/*.{js,ts}", "test/**/*.{js,ts}"],
+  "ignore": ["node_modules", "test/integration/**/*.{js,ts}"]
 }

.vscode/settings.json

@@ -22,5 +22,6 @@
   "[json]": {
     "editor.formatOnSave": true
   },
-  "mochaExplorer.files": "test/**/*.test.{ts,js}"
+  "mochaExplorer.require": "tsx",
+  "mochaExplorer.files": "test/**/*.{ts,js}"
 }

README.md

@@ -101,12 +101,11 @@ Getting up and running is as easy as 1, 2, 3, 4, 5.
    You can also create a **service** using system d.
 
 1. Configure the `config/development.json` and `config/production.json` according to your system settings
-1. Preload the list of Newspaper and of the Topics.
+2. Preload the list of the Topics.
    ```
-   NODE_ENV=development DEBUG=impresso* npm run update-newspapers
    NODE_ENV=development DEBUG=impresso* npm run update-topics
    ```
-1. Start the app! Use the env variable `NODE_ENV` to switch between your development or production configuration file.
+3. Start the app! Use the env variable `NODE_ENV` to switch between your development or production configuration file.
    ```
    NODE_ENV=development DEBUG=impresso* npm run dev
    ```
@@ -205,6 +204,16 @@ When a schema is updated, the typescript types should be regenerated. This can b
 npm run generate-types
 ```
 
+## Projecting new indexed Solr field
+
+A new indexed Solr field should be added to four places:
+
+1. The protobuf definition in the `impresso-jscommons`. A new filter name should be added with its unique ID.
+1. The `SolrMappings` structure in `src/data/constants.ts`. It defines a mapping between the new filter name and the Solr faceting configuration for this filter.
+1. The `solfFilters.yml` file. It defines a mapping between the new filter name and the Solr filter statement builder function.
+1. Optionally: the `stats.yml` file. It defines the statistics datapoint name and the field it maps to. Used to provide faceted data to various graphs and charts.
+
+
 ## Configuration
 
 ### Public API

config/default.example.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "../src/schema/common/config.json",
   "callbackUrls": {
     "passwordReset": "http://localhost:8080/password-reset"
   },
@@ -10,7 +11,7 @@
     "max": 50
   },
   "redis": {
-    "enable": false
+    "host": "localhost"
   },
   "rateLimiter": {
     "enabled": false,
@@ -47,14 +48,6 @@
       "secure": false
     }
   },
-  "neo4j": {
-    "host": "bolt://localhost:7687",
-    "project": "projectname",
-    "auth": {
-      "user": "neo4j",
-      "pass": "neo4j"
-    }
-  },
   "proxy": {
     "host": "http://localhost:3030",
     "iiif": {
@@ -90,23 +83,29 @@
       "users": "auth_user"
     }
   },
-  "solr": {
-    "host": "localhost",
-    "port": 8984,
-    "auth": {
-      "user": "",
-      "pass": "*****"
-    },
-    "search": {
-      "endpoint": "https://.../select"
-    },
-    "mentions": {
-      "endpoint": "https://.../select",
-      "suggest": "https://.../suggest"
-    },
-    "topics": {
-      "endpoint": "https://.../select",
-      "suggest": "https://.../suggest"
-    }
+  "solrConfiguration": {
+    "servers": [
+      {
+        "id": "default",
+        "baseUrl": "https://localhost:8983/solr",
+        "auth": {
+          "read": { "username": "reader", "password": "..." },
+          "write": { "username": "writer", "password": "..." }
+        }
+      }
+    ],
+    "namespaces": [
+      { "namespaceId": "search", "serverId": "default", "index": "search_index" },
+      { "namespaceId": "mentions", "serverId": "default", "index": "mentions_index" },
+      { "namespaceId": "topics", "serverId": "default", "index": "topics_index" },
+      { "namespaceId": "entities", "serverId": "default", "index": "entities_index" },
+      { "namespaceId": "images", "serverId": "default", "index": "images_index" },
+      { "namespaceId": "tr_passages", "serverId": "default", "index": "tr_passages_index" },
+      { "namespaceId": "tr_clusters", "serverId": "default", "index": "tr_clusters_index" },
+      { "namespaceId": "embeddings_de", "serverId": "default", "index": "index_embeddings_de" },
+      { "namespaceId": "embeddings_fr", "serverId": "default", "index": "index_embeddings_fr" },
+      { "namespaceId": "embeddings_lb", "serverId": "default", "index": "index_embeddings_lb" },
+      { "namespaceId": "entities_mentions", "serverId": "default", "index": "mentions_entities_index" }
+    ]
   }
 }

config/default.json

@@ -1,5 +1,7 @@
 {
+  "$schema": "../src/schema/common/config.json",
   "isPublicApi": true,
+  "availablePlans": ["plan-basic", "plan-educational", "plan-researcher"],
   "allowedCorsOrigins": [
     "http://localhost:8080",
     "https://impresso-project.ch/app",
@@ -26,7 +28,7 @@
     "textReuse": { "enabled": true }
   },
   "redis": {
-    "enable": true
+    "host": "localhost"
   },
   "cache": {
     "enabled": true,
@@ -67,14 +69,6 @@
       "secure": true
     }
   },
-  "neo4j": {
-    "host": "bolt://localhost:7687",
-    "project": "impresso",
-    "auth": {
-      "user": "neo4j",
-      "pass": "neo4j"
-    }
-  },
   "media": {
     "host": "http://localhost:3030",
     "path": "/media",
@@ -132,63 +126,30 @@
       "users": "auth_user"
     }
   },
-  "solr": {
-    "host": "localhost",
-    "port": 8984,
-    "auth": {
-      "user": "",
-      "pass": "*****"
-    },
-    "queries": {
-      "hasTextContents": "content_length_i:[1 TO *]"
-    },
-    "search": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "mentions": {
-      "alias": "",
-      "endpoint": "https://.../select",
-      "suggest": "https://.../suggest"
-    },
-    "topics": {
-      "alias": "",
-      "endpoint": "https://.../select",
-      "suggest": "https://.../suggest"
-    },
-    "images": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "entities": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "embeddings_de": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "embeddings_fr": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "embeddings_en": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "embeddings_lb": {
-      "alias": "",
-      "endpoint": "https://.../select"
-    },
-    "tr_passages": {
-      "endpoint": "https://.../select"
-    },
-    "tr_clusters": {
-      "endpoint": "https://.../select"
-    },
-    "entities_mentions": {
-      "endpoint": "https://.../select"
-    }
+  "solrConfiguration": {
+    "servers": [
+      {
+        "id": "default",
+        "baseUrl": "https://localhost:8983/solr",
+        "auth": {
+          "read": { "username": "reader", "password": "..." },
+          "write": { "username": "writer", "password": "..." }
+        }
+      }
+    ],
+    "namespaces": [
+      { "namespaceId": "search", "serverId": "default", "index": "search_index" },
+      { "namespaceId": "mentions", "serverId": "default", "index": "mentions_index" },
+      { "namespaceId": "topics", "serverId": "default", "index": "topics_index" },
+      { "namespaceId": "entities", "serverId": "default", "index": "entities_index" },
+      { "namespaceId": "images", "serverId": "default", "index": "images_index" },
+      { "namespaceId": "tr_passages", "serverId": "default", "index": "tr_passages_index" },
+      { "namespaceId": "tr_clusters", "serverId": "default", "index": "tr_clusters_index" },
+      { "namespaceId": "embeddings_de", "serverId": "default", "index": "index_embeddings_de" },
+      { "namespaceId": "embeddings_fr", "serverId": "default", "index": "index_embeddings_fr" },
+      { "namespaceId": "embeddings_lb", "serverId": "default", "index": "index_embeddings_lb" },
+      { "namespaceId": "entities_mentions", "serverId": "default", "index": "mentions_entities_index" }
+    ]
   },
   "recommender": {
     "byTopics": {

docker-compose.yml

@@ -16,6 +16,14 @@ services:
     ports:
       - 3306:3306
     command: ssh -N cli-mysql-tunnel
+  new-mysql-tunnel:
+    image: kroniak/ssh-client
+    restart: always
+    volumes:
+      - ./docker/config/ssh:/root/.ssh
+    ports:
+      - 3307:3307
+    command: ssh -N cli-new-mysql-tunnel
   # uncomment to run impresso middle layer in docker
   # iml:
   #   image: impresso_middle_layer