Skip to content

Commit 82855b1

Browse files
deepeshabursedeepeshaburse
authored
Merge pull request #771 from wmde/wdqs-separate-concept-uri
2 parents 3dd4b6f + 3eef499 commit 82855b1

File tree

7 files changed

+62
-31
lines changed

7 files changed

+62
-31
lines changed

‎build/wdqs/README.md

+15-12
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,18 @@ When running WDQS in a setup without WDQS-proxy, **please consider disabling the
4040

4141
Variables in **bold** are required.
4242

43-
| Variable | Default | Description |
44-
| ------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
45-
| **`WIKIBASE_HOST`** | "wikibase" | Hostname of the Wikibase service |
46-
| **`WDQS_HOST`** | "wdqs" | WDQS hostname (this service) |
47-
| **`WDQS_PORT`** | "9999" | WDQS port (this service) |
48-
| `WIKIBASE_SCHEME` | "http" | URL scheme of the Wikibase service |
49-
| `WDQS_ENTITY_NAMESPACES` | "120,122" | Wikibase namespaces to load data from |
50-
| `WIKIBASE_MAX_DAYS_BACK` | "90" | Maximum number of days updater can reach back in time from now |
51-
| `MEMORY` | "" | Memory limit for Blazegraph |
52-
| `HEAP_SIZE` | "1g" | Heap size for Blazegraph |
53-
| `BLAZEGRAPH_EXTRA_OPTS` | "" | Extra options to be passed to Blazegraph,they must be prefixed with `-D`. Example: `-Dhttps.proxyHost=http://my.proxy.com -Dhttps.proxyPort=3128`. See [the WDQS User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Configurable_properties). |
43+
| Variable | Default | Description |
44+
| ------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
45+
| **`WIKIBASE_HOST`** | "wikibase" | Hostname to reach the Wikibase service, e.g. the docker network internal hostname |
46+
| **`WIKIBASE_CONCEPT_URI`** | "" | Concept URI, required for `/runUpdate.sh` only, the identifying prefix to entities in this knowledge graph, e.g. the public URL of the Wikibase host. |
47+
| **`WDQS_HOST`** | "wdqs" | WDQS hostname (this service) |
48+
| **`WDQS_PORT`** | "9999" | WDQS port (this service) |
49+
| `WIKIBASE_SCHEME` | "http" | URL scheme used to reach the Wikibase service, e.g. http to reach a local wikibase on the same docker network |
50+
| `WDQS_ENTITY_NAMESPACES` | "120,122" | Wikibase namespaces to load data from |
51+
| `WIKIBASE_MAX_DAYS_BACK` | "90" | Maximum number of days updater can reach back in time from now |
52+
| `MEMORY` | "" | Memory limit for Blazegraph |
53+
| `HEAP_SIZE` | "1g" | Heap size for Blazegraph |
54+
| `BLAZEGRAPH_EXTRA_OPTS` | "" | Extra options to be passed to Blazegraph,they must be prefixed with `-D`. Example: `-Dhttps.proxyHost=http://my.proxy.com -Dhttps.proxyPort=3128`. See [the WDQS User Manual](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Configurable_properties). |
5455

5556
## Example
5657

@@ -143,6 +144,8 @@ services:
143144
nofile:
144145
soft: 32768
145146
hard: 32768
147+
environment:
148+
WIKIBASE_CONCEPT_URI: https://wikibase.example
146149

147150
wdqs-proxy:
148151
image: wikibase/wdqs-proxy
@@ -213,7 +216,7 @@ In the Docker Compose example provided above, you might use the commands and ins
213216
docker compose stop wdqs-updater
214217
215218
# Start an updater with force sync settings
216-
docker compose run --rm wdqs-updater /wdqs/runUpdate.sh -h http://\$WDQS_HOST:\$WDQS_PORT -- --wikibaseUrl \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --conceptUri \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --entityNamespaces \$WDQS_ENTITY_NAMESPACES --init --start $(date +%Y%m%d000000)
219+
docker compose run --rm wdqs-updater /wdqs/runUpdate.sh -h http://\$WDQS_HOST:\$WDQS_PORT -- --wikibaseUrl \$WIKIBASE_SCHEME://\$WIKIBASE_HOST --conceptUri \$WIKIBASE_CONCEPT_URI --entityNamespaces \$WDQS_ENTITY_NAMESPACES --init --start $(date +%Y%m%d000000)
217220
218221
# As soon as you see "Sleeping for 10 secs" in the logs, press CTRL-C to stop it again
219222

‎build/wdqs/runUpdate.sh

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
#!/usr/bin/env bash
22
# This file is provided by the wikibase/wdqs docker image.
33

4+
set +u
5+
if [ -z "$WIKIBASE_CONCEPT_URI" ]; then
6+
echo "WIKIBASE_CONCEPT_URI is required but isn't set.";
7+
exit 1;
8+
fi
9+
set -u
10+
411
cd /wdqs || exit
512

6-
# TODO env vars for entity namespaces, scheme and other settings
713
/wait-for-it.sh "$WIKIBASE_HOST:80" -t 300 -- \
814
/wait-for-it.sh "$WDQS_HOST:$WDQS_PORT" -t 300 -- \
9-
./runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES"
15+
./runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_CONCEPT_URI" --entityNamespaces "$WDQS_ENTITY_NAMESPACES"

‎deploy/docker-compose.yml

+2
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ services:
117117
nofile:
118118
soft: 32768
119119
hard: 32768
120+
environment:
121+
WIKIBASE_CONCEPT_URI: https://${WIKIBASE_PUBLIC_HOST}
120122

121123
wdqs-proxy:
122124
image: wikibase/wdqs-proxy:1

‎test/specs/repo/queryservice.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ describe( 'QueryService', function () {
6969
const itemId = await WikibaseApi.createItem( getTestString( itemLabel ), data );
7070

7171
// query the item using wd: prefix
72-
await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
72+
await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
7373

7474
// wait for WDQS-updater
7575
// eslint-disable-next-line wdio/no-pause
@@ -114,7 +114,7 @@ describe( 'QueryService', function () {
114114
).resolves.toEqual( true );
115115

116116
// query the property using wdt: prefix
117-
await QueryServiceUIPage.open( `SELECT * WHERE{ ?s wdt:${ propertyId } ?o }` );
117+
await QueryServiceUIPage.open( `SELECT * WHERE{ ?s <http://wikibase.example/prop/direct/${ propertyId }> ?o }` );
118118

119119
await QueryServiceUIPage.submit();
120120
await QueryServiceUIPage.resultTable;
@@ -146,7 +146,7 @@ describe( 'QueryService', function () {
146146
).replace( /[()]/g, '' );
147147

148148
// Check it shows up after creation
149-
await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
149+
await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
150150

151151
// wait for WDQS-updater
152152
// eslint-disable-next-line wdio/no-pause
@@ -172,7 +172,7 @@ describe( 'QueryService', function () {
172172
);
173173
await $( '.oo-ui-flaggedElement-destructive button' ).click();
174174

175-
await QueryServiceUIPage.open( `SELECT * WHERE{ wd:${ itemId } ?p ?o }` );
175+
await QueryServiceUIPage.open( `SELECT * WHERE{ <http://wikibase.example/entity/${ itemId }> ?p ?o }` );
176176

177177
// wait for WDQS-updater
178178
// eslint-disable-next-line wdio/no-pause

‎test/specs/repo/special-new-property.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ describe( 'Special:NewProperty', function () {
1717
// eslint-disable-next-line mocha/no-setup-in-describe
1818
dataTypes.forEach( ( dataType: WikibasePropertyType ) => {
1919
// eslint-disable-next-line mocha/no-setup-in-describe
20-
describe( `Should be able to work with datatype ${ dataType.name }`, function () {
20+
describe( `Should be able to work with datatype ${ dataType.name } `, function () {
2121
before( async function () {
2222
if ( dataType.extensionNeeded ) {
2323
await browser.skipIfExtensionNotPresent(
@@ -51,7 +51,7 @@ describe( 'Special:NewProperty', function () {
5151
await browser.waitForJobs();
5252

5353
await expect( browser ).toHaveUrl(
54-
/http:\/\/wikibase\/wiki\/Property:P\d+/
54+
/http:\/\/wikibase.example\/wiki\/Property:P\d+/
5555
);
5656

5757
await expect( $( '.wikibase-propertyview-datatype-value' ) ).toHaveText(

‎test/suites/docker-compose.override.yml

+17
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ services:
2727

2828
wdqs-updater:
2929
image: wikibase/wdqs
30+
environment:
31+
WIKIBASE_CONCEPT_URI: http://${WIKIBASE_PUBLIC_HOST}
3032

3133
wdqs-proxy:
3234
image: wikibase/wdqs-proxy
@@ -55,5 +57,20 @@ services:
5557
SE_NODE_MAX_SESSIONS: ${MAX_INSTANCES}
5658
SE_VNC_NO_PASSWORD: 1
5759

60+
traefik:
61+
command:
62+
# Basic setup
63+
- "--providers.docker=true"
64+
- "--providers.docker.exposedbydefault=false"
65+
# http endpoint
66+
- "--entrypoints.web.address=:80"
67+
- "--entrypoints.web.asdefault"
68+
networks:
69+
default:
70+
aliases:
71+
- wikibase.example
72+
- wdqs-frontend.example
73+
- quickstatements.example
74+
5875
volumes:
5976
wikibase-config:

‎test/test-services.env

+14-11
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
11
# Any of these can be overridden locally in ../local.env
22

3-
# == TEST SERVICES GLOBALS
3+
WIKIBASE_PUBLIC_HOST=wikibase.example
4+
WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend.example
5+
QUICKSTATEMENTS_PUBLIC_HOST=quickstatements.example
46

5-
# URLS
6-
WIKIBASE_URL=http://wikibase
7+
MW_ADMIN_NAME=admin
8+
MW_ADMIN_EMAIL=[email protected]
9+
MW_ADMIN_PASS=change-this-password
10+
11+
# URLs used by tests to access services from within the
12+
# docker network through traefik proxy
13+
WIKIBASE_URL=http://wikibase.example
14+
WDQS_FRONTEND_URL=http://wdqs-frontend.example
15+
QUICKSTATEMENTS_URL=http://quickstatements.example
16+
17+
# URLs used by tests to access services from within the docker network directly
718
WIKIBASE_CLIENT_URL=http://wikibase-client
819
WDQS_URL=http://wdqs:9999
9-
WDQS_FRONTEND_URL=http://wdqs-frontend
1020
WDQS_PROXY_URL=http://wdqs-proxy
11-
QUICKSTATEMENTS_URL=http://quickstatements
12-
13-
# For Reverse Proxy (traefik) service routing
14-
WIKIBASE_PUBLIC_HOST=wikibase
15-
WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend
16-
QUICKSTATEMENTS_PUBLIC_HOST=quickstatements
1721

1822
# CONFIG
19-
2023
# wikibase / wikibase-client / wikibase-jobrunner
2124
MW_SCRIPT_PATH=/w

0 commit comments

Comments
 (0)