Merge pull request #341 from Swirrl/lk/issue_340

Add task to build and publish docker images

Author: scottlowe

.circleci/config.yml

@@ -124,6 +124,18 @@ jobs:
               "${CIRCLE_TAG}" \
               csv2rdf-darwin.tar.gz
 
+  deploy-docker:
+    docker:
+      - image: cimg/clojure:1.11.1
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+    steps:
+      - checkout
+      - run:
+          name: "Deploy csv2rdf docker to public image repository"
+          command: clojure -T:build docker :image-type :registry :to :remote
+
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
 workflows:
@@ -164,3 +176,9 @@ workflows:
               only: /.*/
             branches:
               ignore: /.*/
+      - deploy-docker:
+          context:
+            - gcp-artifact-registry
+            - swirrl-dockerhub-consumer
+          requires:
+            - test

README.md

@@ -47,6 +47,16 @@ need to be provided when processing from a metadata file since the metadata shou
 
     java -jar csv2rdf-standalone.jar -u /path/to/metadata/file.json -o output.ttl
 
+### Running with docker
+
+Docker images are published to the public repository `europe-west2-docker.pkg.dev/swirrl-devops-infrastructure-1/public/csv2rdf`.
+These can be run by specifying the image version to run, and mapping volumes into the container to make local files available within
+the container e.g.
+
+    docker run --rm -v .:/data europe-west2-docker.pkg.dev/swirrl-devops-infrastructure-1/public/csv2rdf:v0.7 -t /data/input.csv -o /data/output.ttl
+
+Note that file paths should be specified relative to the container, not the local system.
+
 ## Using as a library
 
 csv2rdf also exposes its functionality as a library - please see [the csv2rdf library](doc/library.md) for a description of the library and its interface.

build.clj

@@ -1,7 +1,9 @@
 (ns build
   (:require [clojure.tools.build.api :as b]
             [org.corfield.build :as bb]
-            [clojure.string :as str])
+            [clojure.string :as str]
+            [juxt.pack.api :as pack])
+  (:import [com.google.cloud.tools.jib.api JibContainer])
   (:refer-clojure :exclude [test]))
 
 (def lib 'swirrl/csv2rdf)
@@ -63,3 +65,43 @@
               :class-dir class-dir
               :uber-file "target/csv2rdf-app.jar"
               })))
+
+;; A tag name must be valid ASCII and may contain lowercase and uppercase
+;; letters, digits, underscores, periods and dashes
+;; https://docs.docker.com/engine/reference/commandline/tag/#extended-description
+(defn tag [s]
+  (when s (str/replace s #"[^a-zA-Z0-9_.-]" "_")))
+
+(def repo "europe-west2-docker.pkg.dev/swirrl-devops-infrastructure-1/public")
+
+(defn docker [opts]
+  (let [tags (->> ["rev-parse HEAD"
+                   "describe --tags --always"
+                   "branch --show-current"]
+                  (map #(tag (b/git-process {:git-args %})))
+                  (remove nil?))
+        image-type (get opts :image-type :docker)
+        build-args {:basis (b/create-basis {:project "deps.edn" :aliases [:docker]})
+                    ;; If we don't include a tag in the :image-name, then pack implicitly
+                    ;; tags the image with latest, even when we specify additional tags. So
+                    ;; choose a tag arbitrarily to be part of the :image-name, and then
+                    ;; provide the rest in :tags.
+                    :image-name (str repo "/csv2rdf:" (first tags))
+                    :tags (set (rest tags))
+                    :image-type image-type
+                    :base-image "eclipse-temurin:17" ;; An openJDK 17 base docker provided by https://github.com/adoptium/containers#containers
+                    }
+        publish-args {:platforms #{:linux/amd64 :linux/arm64}
+
+                      ;; NOTE Not as documented!
+                      ;; The docstring states that these should be
+                      ;;     :to-registry {:username ... :password ...}
+                      ;; but alas, that is a lie.
+                      ;; https://github.com/juxt/pack.alpha/issues/101
+                      :to-registry-username "_json_key"
+                      :to-registry-password (System/getenv "GCLOUD_SERVICE_KEY")}
+        args (if (= :remote (:to opts))
+               (merge build-args publish-args)
+               build-args)
+        ^JibContainer container (pack/docker args)]
+    (println (.. container (getDigest) (getHash)))))

deps.edn

@@ -30,10 +30,18 @@
             :extra-paths ["test" "test/resources"]
             :main-opts ["-m" "kaocha.runner"]}
 
-           :build {:deps {io.github.seancorfield/build-clj
-                          {:git/tag "v0.6.3" :git/sha "9b8e09b"}}
+           :build {:deps {io.github.seancorfield/build-clj {:git/tag "v0.6.3" :git/sha "9b8e09b"}
+                          io.github.juxt/pack.alpha {:git/sha "802b3d6347376db51093d122eb4b8cf8a7bbd7cf"}
+                          com.google.cloud.tools/jib-core {:mvn/version "0.23.0"}}
+                   
                    :ns-default build}
 
+           :docker {:jvm-opts ["-Xmx4g"
+                               "-Dcom.sun.management.jmxremote.ssl=false"
+                               "-Dcom.sun.management.jmxremote.authenticate=false"
+                               "-Dcom.sun.management.jmxremote.port=3007"]
+                    :main-opts ["-m" "csv2rdf.main"]}
+
            :gen-test-suite {:extra-deps {org.clojure/data.csv {:mvn/version "1.0.1"}}
                             :extra-paths ["test" "test/resources"]
                             :exec-fn csv2rdf.w3c-csvw-suite-test.gen/write-tests-file}}}