diff --git a/pom.xml b/pom.xml index 5d4d51a1..ef5fec78 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,11 @@ repo.jenkins-ci.org https://repo.jenkins-ci.org/public/ + + + jitpack.io + https://jitpack.io + @@ -151,6 +156,19 @@ 1.15 true + + + org.jenkins-ci.plugins + script-security + 1.37 + + + + javax.validation + validation-api + 1.0.0.GA + provided + diff --git a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java index 9321b71f..be0e6d91 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java @@ -35,8 +35,17 @@ import hudson.model.BuildListener; import hudson.tasks.BuildWrapper; import hudson.tasks.BuildWrapperDescriptor; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; + +import java.io.IOException; +import java.io.OutputStream; + +import javax.annotation.CheckForNull; + +import org.kohsuke.stapler.DataBoundSetter; /** + * Logstash note on each output line. * * This BuildWrapper is only a marker and has no other functionality. * The {@link LogstashConsoleLogFilter} uses this BuildWrapper to decide if it should send the log to an indexer. @@ -48,6 +57,9 @@ public class LogstashBuildWrapper extends BuildWrapper { + @CheckForNull + private SecureGroovyScript secureGroovyScript; + /** * Create a new {@link LogstashBuildWrapper}. */ @@ -55,6 +67,11 @@ public class LogstashBuildWrapper extends BuildWrapper public LogstashBuildWrapper() {} + @DataBoundSetter + public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) { + this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null; + } + /** * {@inheritDoc} */ @@ -66,11 +83,26 @@ public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener l { }; } - + @Override - public DescriptorImpl getDescriptor() - { - return (DescriptorImpl)super.getDescriptor(); + public DescriptorImpl getDescriptor() { + return (DescriptorImpl) super.getDescriptor(); + } + + @CheckForNull + public SecureGroovyScript getSecureGroovyScript() { + // FIXME probbably needs to be moved + return secureGroovyScript; + } + + // Method to encapsulate calls for unit-testing + LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { + LogstashScriptProcessor processor = null; + if (secureGroovyScript != null) { + processor = new LogstashScriptProcessor(secureGroovyScript, errorStream); + } + + return new LogstashWriter(build, errorStream, null, build.getCharset(), processor); } /** diff --git a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java index 98b6de92..8092b44b 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java @@ -74,7 +74,6 @@ public Descriptor() { load(); } - @Override public String getDisplayName() { return Messages.DisplayName(); diff --git a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java index 614d63b7..e7ae68e1 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java @@ -30,6 +30,8 @@ import java.io.IOException; import java.io.OutputStream; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Output stream that writes each line to the provided delegate output stream * and also sends it to an indexer for logstash to consume. @@ -54,6 +56,9 @@ LogstashWriter getLogstashWriter() } @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") protected void eol(byte[] b, int len) throws IOException { delegate.write(b, 0, len); this.flush(); @@ -79,6 +84,7 @@ public void flush() throws IOException { */ @Override public void close() throws IOException { + logstash.close(); delegate.close(); super.close(); } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java new file mode 100644 index 00000000..e39f60ea --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java @@ -0,0 +1,50 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc, and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import net.sf.json.JSONObject; + +/** + * Interface describing processors of persisted payload. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public interface LogstashPayloadProcessor { + /** + * Modifies a JSON payload compatible with the Logstash schema. + * + * @param payload the JSON payload that has been constructed so far. + * @return The formatted JSON object, can be null to ignore this payload. + */ + JSONObject process(JSONObject payload) throws Exception; + + /** + * Finalizes any operations, for example returns cashed lines at end of build. + * + * @return A formatted JSON object, can be null when it has nothing. + */ + JSONObject finish() throws Exception; +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java new file mode 100644 index 00000000..a0cf545a --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java @@ -0,0 +1,123 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc. and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.LinkedHashMap; + +import javax.annotation.Nonnull; + +import groovy.lang.Binding; + +import net.sf.json.JSONObject; + +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * This class is handling custom groovy script processing of JSON payload. + * Each call to process executes the script provided in job configuration. + * Script is executed under the same binding each time so that it has ability + * to persist data during build execution if desired by script author. + * When build is finished, script will receive null as the payload and can + * return any cached but non-sent data back for persisting. + * The return value of script is the payload to be persisted unless null. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public class LogstashScriptProcessor implements LogstashPayloadProcessor{ + @Nonnull + private final SecureGroovyScript script; + + @Nonnull + private final OutputStream consoleOut; + + /** Groovy binding for script execution */ + @Nonnull + private final Binding binding; + + /** Classloader for script execution */ + @Nonnull + private final ClassLoader classLoader; + + public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) { + this.script = script; + this.consoleOut = consoleOut; + + // TODO: should we put variables in the binding like manager, job, etc.? + binding = new Binding(); + binding.setVariable("console", new BuildConsoleWrapper()); + + // not sure what the diff is compared to getClass().getClassLoader(); + final Jenkins jenkins = Jenkins.getInstance(); + classLoader = jenkins.getPluginManager().uberClassLoader; + } + + /** + * Helper method to allow logging to build console. + */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") + private void buildLogPrintln(Object o) throws IOException { + consoleOut.write(o.toString().getBytes()); + consoleOut.write("\n".getBytes()); + consoleOut.flush(); + } + + /* + * good examples in: + * https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java + * https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files + */ + @Override + public JSONObject process(JSONObject payload) throws Exception { + binding.setVariable("payload", payload); + script.evaluate(classLoader, binding); + return (JSONObject) binding.getVariable("payload"); + } + + @Override + public JSONObject finish() throws Exception { + buildLogPrintln("Tearing down Script Log Processor.."); + return process(null); + } + + /** + * Helper to allow access from sandboxed script to output messages to console. + */ + private class BuildConsoleWrapper { + @Whitelisted + public void println(Object o) throws IOException { + buildLogPrintln(o); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index c60652c2..a80ddd17 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -37,6 +37,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.annotation.CheckForNull; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; @@ -44,6 +45,8 @@ import java.util.Date; import java.util.List; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * A writer that wraps all Logstash DAOs. Handles error reporting and per build connection state. * Each call to write (one line or multiple lines) sends a Logstash payload to the DAO. @@ -51,6 +54,7 @@ * * @author Rusty Gerard * @author Liam Newman + * @author Aleksandar Kostadinov * @since 1.0.5 */ public class LogstashWriter { @@ -67,11 +71,23 @@ public class LogstashWriter { /* * TODO: the charset must not be transfered to the dao. The dao is shared between different build. */ + @CheckForNull + private final LogstashPayloadProcessor payloadProcessor; + + public LogstashWriter(Run run, OutputStream error, TaskListener listener) { + this(run, error, listener, null); + } + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset) { + this(run, error, listener, charset, null); + } + + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset, LogstashPayloadProcessor payloadProcessor) { this.errorStream = error != null ? error : System.err; this.build = run; this.listener = listener; this.charset = charset; + this.payloadProcessor = payloadProcessor; this.dao = this.getDaoOrNull(); if (this.dao == null) { this.jenkinsUrl = ""; @@ -171,7 +187,35 @@ String getJenkinsUrl() { * Write a list of lines to the indexer as one Logstash payload. */ private void write(List lines) { - JSONObject payload = dao.buildPayload(buildData, jenkinsUrl, lines); + write(dao.buildPayload(buildData, jenkinsUrl, lines)); + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void write(JSONObject payload) { + if (payloadProcessor != null) { + JSONObject processedPayload = payload; + try { + processedPayload = payloadProcessor.process(payload); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error in payload processing.\n"; + + logErrorMessage(msg); + } + if (processedPayload != null) { writeRaw(processedPayload); } + } else { + writeRaw(payload); + } + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void writeRaw(JSONObject payload) { try { dao.push(payload.toString()); } catch (IOException e) { @@ -208,6 +252,9 @@ private LogstashIndexerDao getDaoOrNull() { /** * Write error message to errorStream and set connectionBroken to true. */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private void logErrorMessage(String msg) { try { connectionBroken = true; @@ -219,4 +266,23 @@ private void logErrorMessage(String msg) { } } + + /** + * Signal payload processor that there will be no more lines + */ + public void close() { + if (payloadProcessor != null) { + JSONObject payload = null; + try { + // calling finish() is mandatory to avoid memory leaks + payload = payloadProcessor.finish(); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error with payload processor on finish.\n"; + + logErrorMessage(msg); + } + if (payload != null) writeRaw(payload); + } + } } diff --git a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java index f08d690b..a549c449 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java @@ -26,12 +26,15 @@ import java.nio.charset.Charset; import java.util.Calendar; +import java.util.Date; import java.util.List; import org.apache.commons.lang.StringUtils; import net.sf.json.JSONObject; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Abstract data access object for Logstash indexers. * diff --git a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java index 933ab5b2..5a145d79 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import net.sf.json.JSONObject; +import javax.annotation.CheckForNull; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.FastDateFormat; @@ -56,6 +57,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * POJO for mapping build info to JSON. * @@ -94,6 +97,9 @@ public TestData() { this(null); } + @SuppressFBWarnings( + value="URF_UNREAD_FIELD", + justification="TODO: not sure how to fix this") public TestData(Action action) { AbstractTestResultAction testResultAction = null; if (action instanceof AbstractTestResultAction) { @@ -152,7 +158,7 @@ public List getFailedTests() } private String id; - private String result; + @CheckForNull private String result; private String projectName; private String fullProjectName; private String displayName; @@ -229,6 +235,11 @@ public BuildData(Run build, Date currentTime, TaskListener listener) { } } + // ISO 8601 date format + public static String formatDateIso(Date date) { + return DATE_FORMATTER.format(date); + } + private void initData(Run build, Date currentTime) { Executor executor = build.getExecutor(); @@ -263,7 +274,7 @@ private void initData(Run build, Date currentTime) { } buildDuration = currentTime.getTime() - build.getStartTimeInMillis(); - timestamp = DATE_FORMATTER.format(build.getTimestamp().getTime()); + timestamp = formatDateIso(build.getTime()); } @Override @@ -378,7 +389,7 @@ public String getTimestamp() { } public void setTimestamp(Calendar timestamp) { - this.timestamp = DATE_FORMATTER.format(timestamp.getTime()); + this.timestamp = formatDateIso(timestamp.getTime()); } public String getRootProjectName() { diff --git a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java index e5167f21..5138157c 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java @@ -51,6 +51,8 @@ import jenkins.plugins.logstash.configuration.ElasticSearch; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Elastic Search Data Access Object. * @@ -72,6 +74,9 @@ public ElasticSearchDao(URI uri, String username, String password) { } // Factored for unit testing + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") ElasticSearchDao(HttpClientBuilder factory, URI uri, String username, String password) { if (uri == null) @@ -176,6 +181,9 @@ public void push(String data) throws IOException { } } + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private String getErrorMessage(CloseableHttpResponse response) { ByteArrayOutputStream byteStream = null; PrintStream stream = null; diff --git a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java index 62f8ec86..bbc3d404 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java @@ -32,6 +32,8 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * RabbitMQ Data Access Object. * @@ -109,6 +111,9 @@ public String getPassword() * @see jenkins.plugins.logstash.persistence.LogstashIndexerDao#push(java.lang.String) */ @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") public void push(String data) throws IOException { Connection connection = null; Channel channel = null; diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly new file mode 100644 index 00000000..4c7cbf08 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html new file mode 100644 index 00000000..27404758 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html @@ -0,0 +1,37 @@ +

+ With this script you can filter, modify and/or group messages that are sent to the + configured logstash backend. The following variables are available during execution: +

    +
  • payload - JSONObject contaning the payload to be persisted.
    • +
  • console - a class to output messages to build log without persisting them. It also works in a sandbox: console.println("my logged message").
    • +
+

+

+ The script will be executed once per each line of build console output with + the variable payload set to the complete JSON payload to be + sent including message, timestamp, build url, etc. + The line text in particular will be present under the "message" + key as an array with a single string element. +

+

+ Once script completes execution, the payload variable will be read out + of current Binding and passed down to the Logstash backend to be persisted. If + payload is set to null by the script, then nothing will be + persisted. +

+

+ The script within a build will be executed always using the same Binding. + This means that variables can be saved between script invocations. +

+

+ At the end of the build a payload == null will be submitted. You can use this to + output any messages that you have cached for grouping or other purposes. +

+

+ Example script to filter out some console messages by pattern: + 


+    if (payload && payload["message"][0] =~ /my needless pattern/) {
+      payload = null
+    }
+
+

diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html index fd472c59..d0e26e2b 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html @@ -1,3 +1,6 @@
-

Send individual log lines to Logstash.

+

+ Send individual log lines to Logstash. You can optionally filter and modify them + via groovy script in from advanced configuration. +

diff --git a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java index 13c4774f..e52e08fd 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java @@ -15,6 +15,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.InOrder; import org.mockito.junit.MockitoJUnitRunner; @SuppressWarnings("resource") @@ -122,4 +123,17 @@ public void eolSuccessNoDao() throws Exception { assertEquals("Results don't match", msg, buffer.toString()); verify(mockWriter).isConnectionBroken(); } + + @Test + public void writerClosedBeforeDelegate() throws Exception { + ByteArrayOutputStream mockBuffer = Mockito.spy(buffer); + new LogstashOutputStream(mockBuffer, mockWriter).close(); + + InOrder inOrder = Mockito.inOrder(mockBuffer, mockWriter); + inOrder.verify(mockWriter).close(); + inOrder.verify(mockBuffer).close(); + + // Verify results + assertEquals("Results don't match", "", buffer.toString()); + } } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java index 31cfbc9c..ddb899cb 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java @@ -42,17 +42,47 @@ import hudson.tasks.test.AbstractTestResultAction; import jenkins.plugins.logstash.persistence.BuildData; import jenkins.plugins.logstash.persistence.LogstashIndexerDao; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jvnet.hudson.test.JenkinsRule; import net.sf.json.JSONObject; +import net.sf.json.JSONArray; +import org.mockito.stubbing.Answer; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.Rule; +import org.junit.runner.RunWith; +import org.mockito.*; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; @RunWith(MockitoJUnitRunner.class) public class LogstashWriterTest { + @Rule public JenkinsRule j = new JenkinsRule(); + // Extension of the unit under test that avoids making calls to getInstance() to get the DAO singleton static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, OutputStream error, final String url, final LogstashIndexerDao indexer, - final BuildData data) { - return new LogstashWriter(testBuild, error, null, testBuild.getCharset()) { + final BuildData data, + final LogstashPayloadProcessor processor) { + return new LogstashWriter(testBuild, error, null, testBuild.getCharset(), processor) { @Override LogstashIndexerDao getIndexerDao() { return indexer; @@ -77,6 +107,14 @@ String getJenkinsUrl() { }; } + static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, + OutputStream error, + final String url, + final LogstashIndexerDao indexer, + final BuildData data) { + return createLogstashWriter(testBuild, error, url, indexer, data, null); + } + ByteArrayOutputStream errorBuffer; @Mock LogstashIndexerDao mockDao; @@ -100,7 +138,7 @@ public void before() throws Exception { when(mockBuild.getProject()).thenReturn(mockProject); when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); @@ -123,7 +161,15 @@ public void before() throws Exception { when(mockProject.getFullName()).thenReturn("parent/LogstashWriterTest"); when(mockDao.buildPayload(Matchers.any(BuildData.class), Matchers.anyString(), Matchers.anyListOf(String.class))) - .thenReturn(JSONObject.fromObject("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}")); + .thenAnswer(new Answer() { + @Override + public JSONObject answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + JSONObject json = JSONObject.fromObject("{\"data\":{},\"message\": null,\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + json.element("message", args[2]); + return json; + } + }); Mockito.doNothing().when(mockDao).push(Matchers.anyString()); when(mockDao.getDescription()).thenReturn("localhost:8080"); @@ -159,7 +205,7 @@ public void constructorSuccess() throws Exception { verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getExecutor(); verify(mockBuild, times(2)).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables(); @@ -256,11 +302,56 @@ public void writeBuildLogSuccess() throws Exception { // Verify results // No error output assertEquals("Results don't match", "", errorBuffer.toString()); - verify(mockBuild).getLog(3); + String lines = JSONArray.fromObject(mockBuild.getLog(3)).toString(); + verify(mockBuild, times(2)).getLog(3); verify(mockBuild).getCharset(); verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + verify(mockDao).push("{\"data\":{},\"message\":" + lines + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + } + + @Test + public void writeProcessedSuccess() throws Exception { + String goodMsg = "test message"; + String ignoredMsg = "ignored input"; + String scriptString = + "if (payload) {\n" + + " if (payload['message'][0] =~ /" + ignoredMsg + "/) {\n" + + " payload = null\n" + + " } else {\n" + + " console.println('l');\n" + + " }\n" + + " lastPayload = payload\n" + + "} else {\n" + + " console.println('test build console message')\n" + + " payload = lastPayload\n" + + "}"; + + SecureGroovyScript script = new SecureGroovyScript(scriptString, true, null); + script.configuringWithNonKeyItem(); + LogstashScriptProcessor processor = new LogstashScriptProcessor(script, errorBuffer); + LogstashWriter writer = createLogstashWriter(mockBuild, errorBuffer, "http://my-jenkins-url", mockDao, mockBuildData, processor); + errorBuffer.reset(); + + // Unit under test + writer.write(goodMsg); + writer.write(ignoredMsg); + writer.write(goodMsg); + writer.close(); + + // Verify results + // buffer contains 2 lines logged by the script, then standard tear down message and finally test message at close + assertEquals("Results don't match", "l\nl\nTearing down Script Log Processor..\ntest build console message\n", errorBuffer.toString()); + + InOrder inOrder = Mockito.inOrder(mockDao); + + // first message is generated and pushed to DAO + inOrder.verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + inOrder.verify(mockDao).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + // now message only generated but filtered out by script thus not pushed to DAO + inOrder.verify(mockDao, times(2)).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + // the message at close time is generated by the script so no call to DAO for that + inOrder.verify(mockDao, times(2)).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); verify(mockDao).setCharset(Charset.defaultCharset()); } @@ -327,10 +418,11 @@ public void writeBuildLogGetLogError() throws Exception { List expectedErrorLines = Arrays.asList( "[logstash-plugin]: Unable to serialize log data.", "java.io.IOException: Unable to read log file"); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); verify(mockDao).buildPayload(eq(mockBuildData), eq("http://my-jenkins-url"), logLinesCaptor.capture()); verify(mockDao).setCharset(Charset.defaultCharset()); List actualLogLines = logLinesCaptor.getValue(); + String linesJSON = JSONArray.fromObject(actualLogLines).toString(); + verify(mockDao).push("{\"data\":{},\"message\":" + linesJSON + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); assertThat("The exception was not sent to Logstash", actualLogLines.get(0), containsString(expectedErrorLines.get(0))); assertThat("The exception was not sent to Logstash", actualLogLines.get(1), containsString(expectedErrorLines.get(1))); diff --git a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java index 49435970..e28c15d8 100644 --- a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java +++ b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java @@ -9,7 +9,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -76,7 +75,7 @@ public void before() throws Exception { when(mockBuild.getDescription()).thenReturn("Mock project for testing BuildData"); when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); @@ -132,7 +131,7 @@ private void verifyMocks() throws Exception verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getExecutor(); verify(mockBuild).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables();