First steps toward OBO Dashboard

.gitignore

@@ -1 +1,5 @@
-reports
+reports
+util/__pycache__/
+util/principles/__pycache__/
+build/
+.jekyll-cache/

Jenkinsfile

@@ -0,0 +1,113 @@
+pipeline {
+	agent any
+	// In additional to manual runs, trigger somewhere at midnight to
+	// give us the max time in a day to get things right.
+	triggers {
+		// Master never runs--Feb 31st.
+		cron('0 0 31 2 *')
+		// Nightly @12am.
+		//cron('0 0 2-31 * *')
+		// First of the month @12am, for "release" (also "current").
+		//cron('0 0 1 * *')
+	}
+	environment {
+		///
+		/// Automatic run variables.
+		///
+
+		// Pin dates and day to beginning of run.
+		START_DATE = sh (
+			script: 'date +%Y-%m-%d',
+			returnStdout: true
+		).trim()
+
+		START_DAY = sh (
+			script: 'date +%A',
+			returnStdout: true
+		).trim()
+
+		///
+		/// Internal run variables.
+		///
+
+		// The people to call when things go bad. It is a comma-space
+		// "separated" string.
+		TARGET_ADMIN_EMAILS = '[email protected]'
+	}
+	options{
+		timestamps()
+		buildDiscarder(logRotator(numToKeepStr: '14'))
+	}
+	stages {
+		// Very first: pause for a minutes to give a chance to cancel
+		// and clean the workspace before use.
+		stage('Ready and clean') {
+			steps {
+			// Give us a minute to cancel if we want.
+			//sleep time: 1, unit: 'MINUTES'
+			cleanWs deleteDirs: true, disableDeferredWipeout: true
+			}
+		}
+		stage('Initialize') {
+			steps {
+			// Start preparing environment.
+			sh 'env > env.txt'
+			sh 'echo $BRANCH_NAME > branch.txt'
+			sh 'echo "$BRANCH_NAME"'
+			sh 'cat env.txt'
+			sh 'cat branch.txt'
+			sh 'echo $START_DAY > dow.txt'
+			sh 'echo "$START_DAY"'
+			}
+		}
+		// Our main bit of work.
+		stage('Produce dashboard') {
+			agent {
+				docker {
+					image 'obolibrary/odkfull:v1.1.7'
+					// Reset Jenkins Docker agent default to original
+					// root.
+					args '-u root:root'
+				}
+			}
+			steps {
+				// Create a relative working directory and setup our
+				// working environment.
+				dir('./OBOFoundry.github.io') {
+					git branch: BRANCH_NAME,
+					url: 'https://github.com/OBOFoundry/OBOFoundry.github.io'
+
+					// Setup our environment the way we want.
+					sh 'pip3 install -r requirements.txt'
+					sh 'apt-get -f install lsof'
+					sh 'apt-get -f install zip'
+
+					// Check what our world looks like.
+					sh 'env'
+
+					// We're downloading things, so lets give it a few
+					// tries.
+					timeout(time: 8, unit: 'HOURS') {
+						retry(3){
+							sh 'export PYTHONUNBUFFERED=1'
+							sh 'make clean-dashboard'
+							archiveArtifacts artifacts: 'build/*.zip', onlyIfSuccessful: true
+						}
+					}
+				}
+			}
+		}
+	}
+	post {
+		// Let's let our internal people know if things change.
+		changed {
+			echo "There has been a change in the ${env.BRANCH_NAME} pipeline."
+			mail bcc: '', body: "There has been a pipeline status change in ${env.BRANCH_NAME}. Please see: https://build.obolibrary.io/job/obofoundry/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "OBO pipeline change for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
+		}
+		// Let's let our internal people know if things go badly.
+		failure {
+			echo "There has been a failure in the ${env.BRANCH_NAME} pipeline."
+			mail bcc: '', body: "There has been a pipeline failure in ${env.BRANCH_NAME}. Please see: https://build.obolibrary.io/job/obofoundry/job/pipeline/job/${env.BRANCH_NAME}", cc: '', from: '', replyTo: '', subject: "OBO pipeline FAIL for ${env.BRANCH_NAME}", to: "${TARGET_ADMIN_EMAILS}"
+		}
+	}
+}

Makefile

@@ -63,6 +63,12 @@ tmp:
 reports:
 	mkdir -p $@
 
+reports/robot:
+	mkdir -p $@
+
+reports/principles:
+	mkdir -p $@
+
 
 ### Build Configuration Files
 
@@ -131,6 +137,80 @@ tmp/unsorted-ontologies.yml: $(ONTS) | tmp
 extract-metadata: $(ONTS)
 	./util/extract-metadata.py validate $^
 
+
+### OBO Dashboard
+
+# This is the Jenkins job
+# The reports will be archived
+
+dashboard: build/dashboard.zip
+
+# Build directories
+build:
+	mkdir -p $@
+build/ontologies:
+	mkdir -p $@
+
+# reboot the JVM for Py4J
+reboot:
+	bash ./util/reboot.sh
+
+# This version of ROBOT includes features for starting Py4J
+# This will be changed to ROBOT release once feature is released
+#.PHONY: build/robot.jar
+build/robot.jar: | build
+	curl -o $@ -Lk \
+	https://build.obolibrary.io/job/ontodev/job/robot/job/py4j/lastSuccessfulBuild/artifact/bin/robot.jar
+
+# This version of ROBOT includes features for removing external axioms to create 'base' artefacts
+# This will be removed once this feature is released
+#.PHONY: build/robot-foreign.jar
+build/robot-foreign.jar: | build
+	curl -o $@ -Lk \
+	https://build.obolibrary.io/job/ontodev/job/robot/job/562-feature/lastSuccessfulBuild/artifact/bin/robot.jar
+
+# Generate the initial dashboard results file
+# ALWAYS make sure nothing is running on port 25333
+# Then boot Py4J gateway to ROBOT on that port
+reports/dashboard.csv: registry/ontologies.yml | \
+reports/robot reports/principles build/ontologies build/robot.jar build/robot-foreign.jar
+	make reboot
+	./util/principles/dashboard.py $< $@ --big false
+
+reports/big-dashboard.csv: reports/dashboard.csv
+	make reboot
+	./util/principles/dashboard.py registry/ontologies.yml $@ --big true
+
+# Combine the dashboard files
+reports/dashboard-full.csv: reports/dashboard.csv reports/big-dashboard.csv registry/ontologies.yml
+	./util/principles/sort_tables.py $^ $@
+
+# Generate the HTML grid output for dashboard
+reports/dashboard.html: reports/dashboard-full.csv
+	./util/create-html-grid.py $< $@
+
+# Move all important results to a dashboard directory
+build/dashboard: reports/dashboard.html
+	mkdir -p $@
+	mkdir -p $@/assets
+	mkdir -p $@/reports
+	cp $< $@
+	cp -r reports/robot $@/reports
+	cp -r reports/principles $@/reports
+	cp -r assets/svg $@/assets
+	rm -rf build/dashboard.zip
+	zip -r [email protected] $@
+
+# Clean up, removing ontology files
+# We don't want to keep them because we will download new ones each time to stay up-to-date
+# Reports are all archived in build/dashboard.zip
+clean-dashboard: build/dashboard
+	rm -rf build/ontologies
+	rm -rf reports/robot
+	rm -rf reports/principles
+	rm -rf build/dashboard
+
+
 # Note this should *not* be run as part of general travis jobs, it is expensive
 # and may be prone to false positives as it is inherently network-based
 #

_includes/themes/bootstrap-3/default.html

@@ -20,6 +20,7 @@
 
     <!-- Custom styles -->
     <link href="/css/custom.css" rel="stylesheet" type="text/css" media="all">
+    <link href="/assets/css/pygments.css" rel="stylesheet" type="text/css" media="all">
 
     <style>
 @media (min-width: 768px) {

_layouts/check.html

@@ -0,0 +1,68 @@
+---
+layout: default
+---
+
+{% include navbar.html %}
+<div class="content">    
+
+
+  <div>	
+    <div class="page-header">
+      <h1>
+        Automated Check {{ page.title }}
+      </h1>      
+    <p>
+      {{page.summary}}
+    </p>
+    </div>
+
+  </div>
+
+  <div class="col-md-12">
+    <div class="row">
+      <div class="col-md-3">
+        <ul>
+        {% for p in site.principles %}
+        <li>
+          <a href="../{{p.id}}.html">{{p.title}}</a>
+        </li>
+        {% endfor %}
+        </ul>
+
+        <div class="btn-group" role="group" aria-label="Source">
+          <a href="https://github.com/OBOFoundry/OBOFoundry.github.io/blob/master/principles/{{page.id}}.md">
+            <button type="button" 
+                    data-toggle="tooltip" 
+                    title="See FAQ entry: How I do propose edits to principles?" 
+                    html="true"
+                    class="btn btn-default">
+              View
+            </button>
+          </a>
+          <a href="https://github.com/OBOFoundry/OBOFoundry.github.io/edit/master/principles/checks/{{page.id}}.md">
+
+            <button type="button" 
+                    data-toggle="tooltip" 
+                    title="See FAQ entry: How I do propose edits to principles?" 
+                    html="true"
+                    class="btn btn-default">
+              Edit</button>
+          </a>
+          <!-- <button type="button" class="btn btn-default">Help</button> -->
+        </div>
+        <div>
+          This page is generated via <a href="{{site.repo_src}}_layouts/principle.html">_layouts/check.html</a>. See <a href="/faq/how-do-i-edit-content.html">edit guide</a>
+        </div>
+
+      </div>
+      <div class="col-md-9">
+        {{ content }}
+      </div>
+    </div>
+
+
+  </div>
+
+
+</div>
+