support custom log lines processing via script

pom.xml

@@ -21,6 +21,11 @@
       <id>repo.jenkins-ci.org</id>
       <url>https://repo.jenkins-ci.org/public/</url>
     </repository>
+		<repository>
+        <!-- FIXME remove! -->
+		    <id>jitpack.io</id>
+		    <url>https://jitpack.io</url>
+		</repository>    
   </repositories>
 
   <pluginRepositories>
@@ -151,6 +156,19 @@
     	<version>1.15</version>
   		<optional>true</optional>
     </dependency>
+
+    <dependency>
+       <groupId>org.jenkins-ci.plugins</groupId>
+       <artifactId>script-security</artifactId>
+       <version>1.37</version>
+    </dependency>
+
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>1.0.0.GA</version>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
 
   <build>

src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java

@@ -35,8 +35,17 @@
 import hudson.model.BuildListener;
 import hudson.tasks.BuildWrapper;
 import hudson.tasks.BuildWrapperDescriptor;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import javax.annotation.CheckForNull;
+
+import org.kohsuke.stapler.DataBoundSetter;
 
 /**
+ * Logstash note on each output line.
  *
  * This BuildWrapper is only a marker and has no other functionality.
  * The {@link LogstashConsoleLogFilter} uses this BuildWrapper to decide if it should send the log to an indexer.
@@ -48,13 +57,21 @@
 public class LogstashBuildWrapper extends BuildWrapper
 {
 
+  @CheckForNull
+  private SecureGroovyScript secureGroovyScript;
+
   /**
    * Create a new {@link LogstashBuildWrapper}.
    */
   @DataBoundConstructor
   public LogstashBuildWrapper()
   {}
 
+  @DataBoundSetter
+  public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) {
+    this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null;
+  }
+
   /**
    * {@inheritDoc}
    */
@@ -66,11 +83,26 @@ public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener l
     {
     };
   }
-
+  
   @Override
-  public DescriptorImpl getDescriptor()
-  {
-    return (DescriptorImpl)super.getDescriptor();
+  public DescriptorImpl getDescriptor() {
+    return (DescriptorImpl) super.getDescriptor();
+  }
+
+  @CheckForNull
+  public SecureGroovyScript getSecureGroovyScript() {
+  	// FIXME probbably needs to be moved
+    return secureGroovyScript;
+  }
+
+  // Method to encapsulate calls for unit-testing
+  LogstashWriter getLogStashWriter(AbstractBuild<?, ?> build, OutputStream errorStream) {
+    LogstashScriptProcessor processor = null;
+    if (secureGroovyScript != null) {
+      processor = new LogstashScriptProcessor(secureGroovyScript, errorStream);
+    }
+
+    return new LogstashWriter(build, errorStream, null, build.getCharset(), processor);
   }
 
   /**

src/main/java/jenkins/plugins/logstash/LogstashInstallation.java

@@ -74,7 +74,6 @@ public Descriptor() {
       load();
     }
 
-
     @Override
     public String getDisplayName() {
       return Messages.DisplayName();

src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java

@@ -30,6 +30,8 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 /**
  * Output stream that writes each line to the provided delegate output stream
  * and also sends it to an indexer for logstash to consume.
@@ -54,6 +56,9 @@ LogstashWriter getLogstashWriter()
   }
 
   @Override
+  @SuppressFBWarnings(
+    value="DM_DEFAULT_ENCODING",
+    justification="TODO: not sure how to fix this")
   protected void eol(byte[] b, int len) throws IOException {
     delegate.write(b, 0, len);
     this.flush();
@@ -79,6 +84,7 @@ public void flush() throws IOException {
    */
   @Override
   public void close() throws IOException {
+    logstash.close();
     delegate.close();
     super.close();
   }

src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java

@@ -0,0 +1,50 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc, and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import net.sf.json.JSONObject;
+
+/**
+ * Interface describing processors of persisted payload.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public interface LogstashPayloadProcessor {
+  /**
+   * Modifies a JSON payload compatible with the Logstash schema.
+   *
+   * @param payload the JSON payload that has been constructed so far.
+   * @return The formatted JSON object, can be null to ignore this payload.
+   */
+  JSONObject process(JSONObject payload) throws Exception;
+
+  /**
+   * Finalizes any operations, for example returns cashed lines at end of build.
+   *
+   * @return A formatted JSON object, can be null when it has nothing.
+   */
+  JSONObject finish() throws Exception;
+}

src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java

@@ -0,0 +1,123 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc. and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.LinkedHashMap;
+
+import javax.annotation.Nonnull;
+
+import groovy.lang.Binding;
+
+import net.sf.json.JSONObject;
+
+import jenkins.model.Jenkins;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * This class is handling custom groovy script processing of JSON payload.
+ * Each call to process executes the script provided in job configuration.
+ * Script is executed under the same binding each time so that it has ability
+ * to persist data during build execution if desired by script author.
+ * When build is finished, script will receive null as the payload and can
+ * return any cached but non-sent data back for persisting.
+ * The return value of script is the payload to be persisted unless null.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public class LogstashScriptProcessor implements LogstashPayloadProcessor{
+  @Nonnull
+  private final SecureGroovyScript script;
+
+  @Nonnull
+  private final OutputStream consoleOut;
+
+  /** Groovy binding for script execution */
+  @Nonnull
+  private final Binding binding;
+
+  /** Classloader for script execution */
+  @Nonnull
+  private final ClassLoader classLoader;
+
+  public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) {
+    this.script = script;
+    this.consoleOut = consoleOut;
+
+    // TODO: should we put variables in the binding like manager, job, etc.?
+    binding = new Binding();
+    binding.setVariable("console", new BuildConsoleWrapper());
+
+    // not sure what the diff is compared to getClass().getClassLoader();
+    final Jenkins jenkins = Jenkins.getInstance();
+    classLoader = jenkins.getPluginManager().uberClassLoader;
+  }
+
+  /**
+   * Helper method to allow logging to build console.
+   */
+  @SuppressFBWarnings(
+    value="DM_DEFAULT_ENCODING",
+    justification="TODO: not sure how to fix this")
+  private void buildLogPrintln(Object o) throws IOException {
+    consoleOut.write(o.toString().getBytes());
+    consoleOut.write("\n".getBytes());
+    consoleOut.flush();
+  }
+
+  /*
+   * good examples in:
+   *  https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java
+   *  https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files
+   */
+  @Override
+  public JSONObject process(JSONObject payload) throws Exception {
+    binding.setVariable("payload", payload);
+    script.evaluate(classLoader, binding);
+    return (JSONObject) binding.getVariable("payload");
+  }
+
+  @Override
+  public JSONObject finish() throws Exception {
+    buildLogPrintln("Tearing down Script Log Processor..");
+    return process(null);
+  }
+
+  /**
+   * Helper to allow access from sandboxed script to output messages to console.
+   */
+  private class BuildConsoleWrapper {
+    @Whitelisted
+    public void println(Object o) throws IOException {
+      buildLogPrintln(o);
+    }
+  }
+}