Skip to content

Commit e8cf107

Browse files
jamesaovertonjamesaoverton
authored
Merge pull request #1069 from OBOFoundry/dashboard
First steps toward OBO Dashboard
2 parents 6060a90 + 6e7ec22 commit e8cf107

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+4622
-63
lines changed

‎.gitignore

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
1-
reports
1+
reports
2+
util/__pycache__/
3+
util/principles/__pycache__/
4+
build/
5+
.jekyll-cache/

‎Jenkinsfile

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
pipeline {
2+
agent any
3+
// In additional to manual runs, trigger somewhere at midnight to
4+
// give us the max time in a day to get things right.
5+
triggers {
6+
// Master never runs--Feb 31st.
7+
cron('0 0 31 2 *')
8+
// Nightly @12am.
9+
//cron('0 0 2-31 * *')
10+
// First of the month @12am, for "release" (also "current").
11+
//cron('0 0 1 * *')
12+
}
13+
environment {
14+
///
15+
/// Automatic run variables.
16+
///
17+
18+
// Pin dates and day to beginning of run.
19+
START_DATE = sh (
20+
script: 'date +%Y-%m-%d',
21+
returnStdout: true
22+
).trim()
23+
24+
START_DAY = sh (
25+
script: 'date +%A',
26+
returnStdout: true
27+
).trim()
28+
29+
///
30+
/// Internal run variables.
31+
///
32+
33+
// The people to call when things go bad. It is a comma-space
34+
// "separated" string.
35+
TARGET_ADMIN_EMAILS = '[email protected]'
36+
}
37+
options{
38+
timestamps()
39+
buildDiscarder(logRotator(numToKeepStr: '14'))
40+
}
41+
stages {
42+
// Very first: pause for a minutes to give a chance to cancel
43+
// and clean the workspace before use.
44+
stage('Ready and clean') {
45+
steps {
46+
// Give us a minute to cancel if we want.
47+
//sleep time: 1, unit: 'MINUTES'
48+
cleanWs deleteDirs: true, disableDeferredWipeout: true
49+
}
50+
}
51+
stage('Initialize') {
52+
steps {
53+
// Start preparing environment.
54+
sh 'env > env.txt'
55+
sh 'echo $BRANCH_NAME > branch.txt'
56+
sh 'echo "$BRANCH_NAME"'
57+
sh 'cat env.txt'
58+
sh 'cat branch.txt'
59+
sh 'echo $START_DAY > dow.txt'
60+
sh 'echo "$START_DAY"'
61+
}
62+
}
63+
// Our main bit of work.
64+
stage('Produce dashboard') {
65+
agent {
66+
docker {
67+
image 'obolibrary/odkfull:v1.1.7'
68+
// Reset Jenkins Docker agent default to original
69+
// root.
70+
args '-u root:root'
71+
}
72+
}
73+
steps {
74+
// Create a relative working directory and setup our
75+
// working environment.
76+
dir('./OBOFoundry.github.io') {
77+
git branch: BRANCH_NAME,
78+
url: 'https://github.com/OBOFoundry/OBOFoundry.github.io'
79+
80+
// Setup our environment the way we want.
81+
sh 'pip3 install -r requirements.txt'
82+
sh 'apt-get -f install lsof'
83+
sh 'apt-get -f install zip'
84+
85+
// Check what our world looks like.
86+
sh 'env'
87+
88+
// We're downloading things, so lets give it a few
89+
// tries.
90+
timeout(time: 8, unit: 'HOURS') {
91+
retry(3){
92+
sh 'export PYTHONUNBUFFERED=1'
93+
sh 'make clean-dashboard'
94+
archiveArtifacts artifacts: 'build/*.zip', onlyIfSuccessful: true
95+
}
96+
}
97+
}
98+
}
99+
}
100+
}
101+
post {
102+
// Let's let our internal people know if things change.
103+
changed {
104+
echo "There has been a change in the ${env.BRANCH_NAME} pipeline."
105+
mail bcc: '', body: "There has been a pipeline status change in ${env.BRANCH_NAME}. Please see: https://build.obolibrary.io/job/obofoundry/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "OBO pipeline change for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
106+
}
107+
// Let's let our internal people know if things go badly.
108+
failure {
109+
echo "There has been a failure in the ${env.BRANCH_NAME} pipeline."
110+
mail bcc: '', body: "There has been a pipeline failure in ${env.BRANCH_NAME}. Please see: https://build.obolibrary.io/job/obofoundry/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "OBO pipeline FAIL for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
111+
}
112+
}
113+
}

‎Makefile

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ tmp:
6363
reports:
6464
mkdir -p $@
6565

66+
reports/robot:
67+
mkdir -p $@
68+
69+
reports/principles:
70+
mkdir -p $@
71+
6672

6773
### Build Configuration Files
6874

@@ -131,6 +137,80 @@ tmp/unsorted-ontologies.yml: $(ONTS) | tmp
131137
extract-metadata: $(ONTS)
132138
./util/extract-metadata.py validate $^
133139

140+
141+
### OBO Dashboard
142+
143+
# This is the Jenkins job
144+
# The reports will be archived
145+
146+
dashboard: build/dashboard.zip
147+
148+
# Build directories
149+
build:
150+
mkdir -p $@
151+
build/ontologies:
152+
mkdir -p $@
153+
154+
# reboot the JVM for Py4J
155+
reboot:
156+
bash ./util/reboot.sh
157+
158+
# This version of ROBOT includes features for starting Py4J
159+
# This will be changed to ROBOT release once feature is released
160+
#.PHONY: build/robot.jar
161+
build/robot.jar: | build
162+
curl -o $@ -Lk \
163+
https://build.obolibrary.io/job/ontodev/job/robot/job/py4j/lastSuccessfulBuild/artifact/bin/robot.jar
164+
165+
# This version of ROBOT includes features for removing external axioms to create 'base' artefacts
166+
# This will be removed once this feature is released
167+
#.PHONY: build/robot-foreign.jar
168+
build/robot-foreign.jar: | build
169+
curl -o $@ -Lk \
170+
https://build.obolibrary.io/job/ontodev/job/robot/job/562-feature/lastSuccessfulBuild/artifact/bin/robot.jar
171+
172+
# Generate the initial dashboard results file
173+
# ALWAYS make sure nothing is running on port 25333
174+
# Then boot Py4J gateway to ROBOT on that port
175+
reports/dashboard.csv: registry/ontologies.yml | \
176+
reports/robot reports/principles build/ontologies build/robot.jar build/robot-foreign.jar
177+
make reboot
178+
./util/principles/dashboard.py $< $@ --big false
179+
180+
reports/big-dashboard.csv: reports/dashboard.csv
181+
make reboot
182+
./util/principles/dashboard.py registry/ontologies.yml $@ --big true
183+
184+
# Combine the dashboard files
185+
reports/dashboard-full.csv: reports/dashboard.csv reports/big-dashboard.csv registry/ontologies.yml
186+
./util/principles/sort_tables.py $^ $@
187+
188+
# Generate the HTML grid output for dashboard
189+
reports/dashboard.html: reports/dashboard-full.csv
190+
./util/create-html-grid.py $< $@
191+
192+
# Move all important results to a dashboard directory
193+
build/dashboard: reports/dashboard.html
194+
mkdir -p $@
195+
mkdir -p $@/assets
196+
mkdir -p $@/reports
197+
cp $< $@
198+
cp -r reports/robot $@/reports
199+
cp -r reports/principles $@/reports
200+
cp -r assets/svg $@/assets
201+
rm -rf build/dashboard.zip
202+
zip -r $@.zip $@
203+
204+
# Clean up, removing ontology files
205+
# We don't want to keep them because we will download new ones each time to stay up-to-date
206+
# Reports are all archived in build/dashboard.zip
207+
clean-dashboard: build/dashboard
208+
rm -rf build/ontologies
209+
rm -rf reports/robot
210+
rm -rf reports/principles
211+
rm -rf build/dashboard
212+
213+
134214
# Note this should *not* be run as part of general travis jobs, it is expensive
135215
# and may be prone to false positives as it is inherently network-based
136216
#

‎_includes/themes/bootstrap-3/default.html

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
<!-- Custom styles -->
2222
<link href="/css/custom.css" rel="stylesheet" type="text/css" media="all">
23+
<link href="/assets/css/pygments.css" rel="stylesheet" type="text/css" media="all">
2324

2425
<style>
2526
@media (min-width: 768px) {

‎_layouts/check.html

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
---
2+
layout: default
3+
---
4+
5+
{% include navbar.html %}
6+
<div class="content">
7+
8+
9+
<div>
10+
<div class="page-header">
11+
<h1>
12+
Automated Check {{ page.title }}
13+
</h1>
14+
<p>
15+
{{page.summary}}
16+
</p>
17+
</div>
18+
19+
</div>
20+
21+
<div class="col-md-12">
22+
<div class="row">
23+
<div class="col-md-3">
24+
<ul>
25+
{% for p in site.principles %}
26+
<li>
27+
<a href="../{{p.id}}.html">{{p.title}}</a>
28+
</li>
29+
{% endfor %}
30+
</ul>
31+
32+
<div class="btn-group" role="group" aria-label="Source">
33+
<a href="https://github.com/OBOFoundry/OBOFoundry.github.io/blob/master/principles/{{page.id}}.md">
34+
<button type="button"
35+
data-toggle="tooltip"
36+
title="See FAQ entry: How I do propose edits to principles?"
37+
html="true"
38+
class="btn btn-default">
39+
View
40+
</button>
41+
</a>
42+
<a href="https://github.com/OBOFoundry/OBOFoundry.github.io/edit/master/principles/checks/{{page.id}}.md">
43+
44+
<button type="button"
45+
data-toggle="tooltip"
46+
title="See FAQ entry: How I do propose edits to principles?"
47+
html="true"
48+
class="btn btn-default">
49+
Edit</button>
50+
</a>
51+
<!-- <button type="button" class="btn btn-default">Help</button> -->
52+
</div>
53+
<div>
54+
This page is generated via <a href="{{site.repo_src}}_layouts/principle.html">_layouts/check.html</a>. See <a href="/faq/how-do-i-edit-content.html">edit guide</a>
55+
</div>
56+
57+
</div>
58+
<div class="col-md-9">
59+
{{ content }}
60+
</div>
61+
</div>
62+
63+
64+
</div>
65+
66+
67+
</div>
68+

0 commit comments

Comments
 (0)