Release 3.1.0

Author: iankrintz

.gitlab-ci.yml

@@ -4,6 +4,16 @@ import com.moowork.gradle.node.task.NodeTask
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
 import org.gradle.api.tasks.testing.logging.TestLogEvent.*
 
+buildscript {
+  repositories {
+    if (gitLabCICD) {
+      maven { url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle-plugins/") }
+    }
+  }
+  extra.apply{ set("gitLabCICD", gitLabCICD) }
+}
+
+
 plugins {
     `kotlin-dsl`
 
@@ -52,7 +62,7 @@ plugins {
     // https://docs.spring.io/spring-boot/docs/current/gradle-plugin/reference/html/
     // - A Gradle plugin that allows you to package executable jar or war archives,
     //   run Spring Boot applications, and use the dependency management provided by spring-boot-dependencies
-    id("org.springframework.boot").version("2.7.9").apply(false)
+    id("org.springframework.boot").version("2.7.12").apply(false)
 
     // Gogradle plugin
     // https://github.com/gogradle/gogradle
@@ -61,9 +71,13 @@ plugins {
 
 }
 
-// resolve build dependencies from Bintray
 repositories {
-    mavenCentral()
+    if (gitLabCICD) {
+      maven(url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle.mavencentral/"))
+      maven(url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle-plugins/"))
+    } else {
+      mavenCentral()
+    }
 }
 
 subprojects {
@@ -138,11 +152,22 @@ dependencyCheck {
 }
 
 allprojects {
-    // resolve all subproject dependencies from these repos
+    // resolve subproject dependencies through repos based on build environment
     repositories {
+      if (gitLabCICD) {
+        maven(url = "https://artifacts.ncei.noaa.gov/artifactory/spring-milestone/")
+        maven(url = "https://artifacts.ncei.noaa.gov/artifactory/confluent-maven/")
+
+        maven(url = "https://artifacts.ncei.noaa.gov/artifactory/gradle.mavencentral/")
+        maven(url = "https://artifacts.ncei.noaa.gov/artifactory/gradle-plugins/")
+      } else {
         mavenCentral()
-        maven(url= "https://repo.spring.io/milestone")
+        maven(url = "https://repo.spring.io/milestone")
         maven(url = "https://packages.confluent.io/maven/")
+      }
+      // Switch based on artifactory access. Check logs during build to ensure jitpack isn't used on-prem.
+      if (System.getenv("ARTIFACTORY_API_KEY") != null) {
+        project.logger.lifecycle("Using NCEI Artifactory for dependency resolution")
         maven {
           name = "NCEI_MAVEN_PROD"
           url = uri("https://artifacts.ncei.noaa.gov/artifactory/ncei-maven/")
@@ -154,7 +179,10 @@ allprojects {
             create<HttpHeaderAuthentication>("header")
           }
         }
-
+      } else {
+        project.logger.lifecycle("*** WARNING: using jitpack for dependency resolution. This is NOT allowed for NCEI on-prem builds ***")
+        maven(url = "https://jitpack.io")
+      }
     }
 }
 
@@ -249,7 +277,14 @@ subprojects {
         // apply a common node/npm version to all projects using node
         configure<com.moowork.gradle.node.NodeExtension> {
             version = Versions.NODE
-            npmVersion = Versions.NPM
+            // pull the node dist from artifactory if in GitLab CI/CD
+            if (gitLabCICD) {
+              distBaseUrl = "https://artifacts.ncei.noaa.gov/artifactory/node-dist"
+              // npmVersion is intentionally not set; use the one bundled with node
+              // since we cannot set the url to artifactory for the npm dist
+            } else {
+              npmVersion = Versions.NPM
+            }
             workDir = file("${rootProject.buildDir}/nodejs")
             npmWorkDir = file("${rootProject.buildDir}/npm")
             nodeModulesDir = file("${project.projectDir}")
@@ -305,6 +340,16 @@ subprojects {
         configurations.all {
             resolutionStrategy.eachDependency {
 
+                if (requested.group == "org.xerial.snappy" && requested.name == "snappy-java") {
+                      useVersion("1.1.10.1")
+                      because("override version since kafka-clients only use up to version 1.1.8.4")
+                }
+
+                if (requested.group == "org.mozilla" && requested.name == "rhino") {
+                      useVersion("1.7.14")
+                      because("override version since json-schema-validator only uses up to version 1.7.7.2")
+                }
+
                 if (requested.group == "com.github.everit-org.json-schema" && requested.name == "org.everit.json.schema") {
                   useTarget("com.github.erosb:everit-json-schema:1.14.2")
                 }
@@ -400,14 +445,14 @@ subprojects {
                 }
                 if (requested.group.startsWith("org.apache.tomcat") &&
                         requested.name.contains("tomcat") &&
-                        requested.version!! < "9.0.68") {
-                    useVersion("9.0.68")
+                        requested.version!! < "9.0.75") {
+                    useVersion("9.0.75")
                     because("Enforce tomcat 9.0.58+ to avoid vulnerabilities CVE-2022-23181\n" +
                             "9.0.68+ to avoid CVE-2022-34305")
                 }
                 if (requested.group.startsWith("org.apache.tomcat.embed") &&
-                    requested.version!! < "9.0.58") {
-                    useVersion("9.0.58")
+                    requested.version!! < "9.0.75") {
+                    useVersion("9.0.75")
                     because("Fixes CVE-2022-23181")
                 }
                 if (requested.group.startsWith("io.netty.incubator") &&

build.gradle.kts

@@ -2,9 +2,16 @@ plugins {
     `kotlin-dsl`
 }
 
+val gitLabCICD: Boolean = System.getProperty("gitLabCICD").toBoolean()
+
 repositories {
-    mavenCentral()
-    maven { url = uri("https://jcenter.bintray.com") }
+    if (gitLabCICD) {
+      maven { url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle-plugins/") }
+      maven { url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle.mavencentral/") }
+    } else {
+      mavenCentral()
+      maven { url = uri("https://jcenter.bintray.com") }
+    }
 }
 
 dependencies {

buildSrc/build.gradle.kts

@@ -0,0 +1,10 @@
+pluginManagement {
+  repositories {
+    if (System.getProperty("gitLabCICD").toBoolean()) {
+      maven { url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle-plugins/") }
+      maven { url = uri("https://artifacts.ncei.noaa.gov/artifactory/gradle.mavencentral/") }
+    } else {
+      // plugins resolved elsewhere?
+    }
+  }
+}

buildSrc/settings.gradle.kts

@@ -18,7 +18,7 @@ object Versions {
     // https://github.com/opencontainers/image-spec/blob/master/annotations.md#annotations
     const val LABEL_SCHEMA: String = "1.0"
 
-    const val NODE: String = "14.17.0"
+    const val NODE: String = "16.19.1"
     const val NPM: String = "8.4.1"
 
     const val ELASTIC: String = "7.17.9"
@@ -44,8 +44,13 @@ object Versions {
     const val JACKSON_CORE = "2.14.2" // A lot of other dependencies bring this in though.
 
     const val ONESTOP_SCHEMAS: String = "0.7.6"
+
+    // From schemas subproject
+    const val COMMONS_TEXT = "1.10.0"
 }
 
+val gitLabCICD: Boolean = System.getProperty("gitLabCICD").toBoolean()
+
 // data classes
 data class Author(
         val name: String,
@@ -59,9 +64,6 @@ fun environment(variable: String, default: String = ""): String {
 }
 
 
-
-
-
 fun formatAuthors(authors: Collection<Author>, pretty: Boolean = false): String {
     // Your Name <[email protected]> (http://example.com)
     val multipleAuthors: Boolean = authors.size > 1

buildSrc/src/main/kotlin/utils.kt

@@ -7,7 +7,11 @@ task<com.moowork.gradle.node.npm.NpmTask>("formatCheck") {
 }
 
 task<com.moowork.gradle.node.npm.NpmTask>("retire") {
-    setArgs(mutableListOf("run", "retire"))
+    if (gitLabCICD) {
+      setArgs(mutableListOf("run", "retireGitLabCICD"))
+    } else {
+      setArgs(mutableListOf("run", "retire"))
+    }
 }
 
 tasks.getByName("check") {

client/build.gradle.kts

@@ -14,9 +14,11 @@
     "build": "npm install && NODE_ENV=production ./node_modules/.bin/webpack --mode production",
     "clean": "rm -rf node_modules && rm -rf build && rm -rf coverage",
     "retire": "retire -p",
+    "retireGitLabCICD": "retire -p --jsrepo https://artifacts.ncei.noaa.gov/artifactory/jsretire-repository/jsrepository.json --noderepo https://artifacts.ncei.noaa.gov/artifactory/jsretire-repository/npmrepository.json --cacert /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
     "dependencies": "npm list -depth=8",
     "auditFix": "npm audit fix --force",
-    "audit": "npm audit --trace-deprecation"
+    "audit": "npm audit --trace-deprecation",
+    "updateBrowsers": "echo updating browsers;npx browserslist@latest --update-db"
   },
   "dependencies": {
     "@chakra-ui/react": "^1.8.3",

client/package-lock.json

@@ -1,4 +1,5 @@
 import _ from 'lodash'
+import Immutable from 'seamless-immutable'
 import {fetchGranuleSearch} from './AsyncHelpers'
 import {
   assembleSearchRequest,
@@ -235,7 +236,11 @@ const granulesForCartPromise = (
 
   // Ensure the filter page offset is 0, since we are grabbing as much as we can
   // from the beginning -- up to `maxCartAddition` in a single request.
-  const filterStateModifiedForCartRequest = granuleFilter.set('pageOffset', 0)
+  const filterStateModifiedForCartRequest = Immutable.set(
+    granuleFilter,
+    'pageOffset',
+    0
+  )
 
   // generate the request body based on filters, and if we need facets or not
   const body = granuleBodyBuilder(

client/package.json

@@ -214,11 +214,11 @@ export default class DescriptionView extends React.Component {
     const xmlLink = (
       <a
         href={
-          getApiRegistryPath() + '/metadata/collection/' + itemUuid + '/raw/xml'
+          getApiRegistryPath() + '/metadata/collection/' + itemUuid + '/raw'
         }
         target={'_blank'}
       >
-        Download full XML metadata here
+        Download the full metadata here
       </a>
     )
 

client/src/actions/routing/GranuleSearchRouteActions.js

@@ -1,5 +1,11 @@
 import React, {useRef, useState} from 'react'
-import {animated, useChain, useSpring, config} from 'react-spring'
+import {
+  animated,
+  useChain,
+  useSpring,
+  useTransition,
+  config,
+} from 'react-spring'
 import {useMeasure, usePrevious} from '../../../effects/CommonEffects'
 
 export const DIRECTION_DOWN = 'down'
@@ -122,11 +128,7 @@ const Drawer = ({
   })
 
   // chain animations and reverse the chain when drawer is closing
-  useChain(
-    drawer.open
-      ? [ {current: downRef.current}, {current: overRef.current} ]
-      : [ {current: overRef.current}, {current: downRef.current} ]
-  )
+  useChain(drawer.open ? [ downRef, overRef ] : [ overRef, downRef ])
 
   const styleContent = {
     overflow: 'hidden',

client/src/components/collections/detail/DescriptionView.jsx

@@ -62,7 +62,7 @@ class TopicsMenu extends React.Component {
     ]
     topics = topics.map((topic, i) => {
       return (
-        <li>
+        <li key={i}>
           <TopicsMenuButton key={i} topic={topic} onClick={this.search} />
         </li>
       )

client/src/components/common/ui/Drawer.jsx

@@ -89,10 +89,10 @@ export default function GranuleItem(props){
 
   const xmlLink = (
     <a
-      href={getApiRegistryPath() + '/metadata/granule/' + itemId + '/raw/xml'}
+      href={getApiRegistryPath() + '/metadata/granule/' + itemId + '/raw'}
       target={'_blank'}
     >
-      XML
+      Download raw metadata
     </a>
   )
 

client/src/components/landing/TopicsMenu.jsx

@@ -120,7 +120,7 @@ public Builder setTitle(String title) {
     }
 
     /**
-     * A human-readable explanation specific to this occurrence of the problem. Like title, this field’s value can be localized.
+     * A human-readable explanation specific to this occurrence of the problem. Like title, the value of this field can be localized.
      * @param detail String
      * @return
      */
@@ -141,7 +141,7 @@ public Builder setMeta(JsonApiMeta meta) {
 
     /**
      * An object containing references to the source of the error,
-     * @param source ErrorSource An object containing references to the source of the error,
+     * @param source ErrorSource An object containing references to the source of the error.
      * @return
      */
     public Builder setSource(ErrorSource source) {

client/src/components/results/granules/GranuleItem.jsx

@@ -2,9 +2,6 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer
 import org.testcontainers.utility.DockerImageName
 
 buildscript {
-  repositories {
-    mavenCentral()
-  }
   dependencies {
     classpath "org.testcontainers:elasticsearch:${Versions.TEST_CONTAINERS}"
     classpath "org.testcontainers:kafka:${Versions.TEST_CONTAINERS}"

data-common/src/main/java/org/cedar/onestop/data/api/JsonApiError.java

@@ -48,7 +48,7 @@ public IndexerApp(AppConfig appConfig) throws IOException {
     streamsProps = KafkaHelpers.buildStreamsConfig(appConfig);
     streamsApp = new KafkaStreams(streamsTopology, streamsProps);
     var maxFailures = appConfig.getOrDefault("streams.exception.max.failures", 2, Integer.class);
-    var maxTimeInterval = appConfig.getOrDefault("streams.exception.max.time.millis", 3600000L, Long.class);
+    var maxTimeInterval = appConfig.getOrDefault("streams.exception.max.time.millis", 3600000, Integer.class);
     exceptionHandler = new UncaughtExceptionHandler(maxFailures, maxTimeInterval);
     probeServer = new KafkaHealthProbeServer(streamsApp);
   }

indexer/build.gradle

@@ -20,7 +20,7 @@ dependencies {
 
   // reactor release train BOM governs reactor dependency versions
   implementation(platform("io.projectreactor:reactor-bom:${Versions.REACTOR_BOM}"))
-  implementation("io.projectreactor.netty:reactor-netty:1.0.28")
+  implementation("io.projectreactor.netty:reactor-netty:1.0.33")
 
   testImplementation("org.codehaus.groovy:groovy:${Versions.GROOVY}")
   testImplementation("org.spockframework:spock-core:${Versions.SPOCK}")
@@ -36,7 +36,7 @@ dependencies {
   testImplementation("io.confluent:kafka-schema-registry:${Versions.CONFLUENT}:tests")
   testImplementation("io.confluent:kafka-streams-avro-serde:${Versions.CONFLUENT}")
   testImplementation("com.github.everit-org.json-schema:org.everit.json.schema:1.9.2")
-  testImplementation("org.json:json:20180813")
+  testImplementation("org.json:json:20230227")
 }
 
 jar {