feat: Use Apache Common Compressor API to [de]compress serialized models (#2165)

Since #2094 and #2103 we can [de]serialize the model (CtFactory) with standard java serialization in a file, or a gzip file to save space. Choice was made to write things so that we can add new compression types easily. 

In this PR, I propose to use Apache Common Compressor API (+other files needed for some compression types such as LZMA).

Main changes in src/main/java/spoon/support/SerializationModelStreamer.java and its junit test case src/test/java/spoon/test/serializable/ModelStreamerTest.java and of course pom.xml.

I don't know what you think about adding a new lib, but this lib allows very easily (i) to test what compressor is used for an inputstream (except 7zip btw), (ii) to decompress, (iii) to compress.

Looking forward discussing it with you guyz :)
Cheers,
Mehdi

Author: Mehdi Kaytoue

pom.xml

@@ -268,11 +268,23 @@
       <artifactId>commons-lang3</artifactId>
       <version>3.5</version>
     </dependency>
+    <!-- https://mvnrepository.com/artifact/org.tukaani/xz -->
+	<dependency>
+    	<groupId>org.tukaani</groupId>
+    	<artifactId>xz</artifactId>
+    	<version>1.8</version>
+	</dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>2.9.5</version>
     </dependency>
+    <dependency>
+      <!-- support for compressed serialized ASTs -->
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.17</version>
+	</dependency>
     <dependency>
       <!-- to reproduce JTD error with nullable annotation -->
       <groupId>com.google.guava</groupId>

src/main/java/spoon/support/CompressionType.java

@@ -21,8 +21,8 @@
  *	@see SerializationModelStreamer
  */
 public enum CompressionType {
-
 	NONE,
-	GZIP;
-
+	GZIP,
+	LZMA,
+	BZIP2;
 }

src/main/java/spoon/support/SerializationModelStreamer.java

@@ -26,6 +26,13 @@
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
+import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream;
+
 import spoon.Launcher;
 import spoon.reflect.ModelStreamer;
 import spoon.reflect.declaration.CtElement;
@@ -48,6 +55,10 @@ public SerializationModelStreamer() {
 	public void save(Factory f, OutputStream out) throws IOException {
 		if (f.getEnvironment().getCompressionType() == CompressionType.GZIP) {
 			out = new GZIPOutputStream(out);
+		} else if (f.getEnvironment().getCompressionType() == CompressionType.LZMA) {
+			out = new LZMACompressorOutputStream(out);
+		} else if (f.getEnvironment().getCompressionType() == CompressionType.BZIP2) {
+			out = new BZip2CompressorOutputStream(out);
 		}
 		try (ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(out))) {
 			oos.writeObject(f);
@@ -57,19 +68,21 @@ public void save(Factory f, OutputStream out) throws IOException {
 
 	@Override
 	public Factory load(InputStream in) throws IOException {
-		if (!in.markSupported()) {
-			in = new BufferedInputStream(in);
-		}
-		// Check if it is a GZIP
-		in.mark(2);
-		int ch1 = in.read();
-		int ch2 = in.read();
-		int header = ((ch2 << 8) + (ch1 << 0));
-		in.reset();
-		if (header == GZIPInputStream.GZIP_MAGIC) {
-			in = new GZIPInputStream(in);
-		}
-		try (ObjectInputStream ois = new ObjectInputStream(in)) {
+		try {
+			BufferedInputStream buffered = new BufferedInputStream(in);
+			try {
+				String s = CompressorStreamFactory.detect(buffered);
+				if (s.equals(CompressorStreamFactory.GZIP)) {
+					in = new GZIPInputStream(buffered);
+				} else if (s.equals(CompressorStreamFactory.LZMA)) {
+					in = new LZMACompressorInputStream(buffered);
+				} else if (s.equals(CompressorStreamFactory.BZIP2)) {
+					in = new BZip2CompressorInputStream(buffered);
+				}
+			} catch (CompressorException e) {
+				in = buffered;
+			}
+			ObjectInputStream ois = new ObjectInputStream(in);
 			final Factory f = (Factory) ois.readObject();
 			//create query using factory directly
 			//because any try to call CtElement#map or CtElement#filterChildren will fail on uninitialized factory

src/test/java/spoon/test/serializable/ModelStreamerTest.java

@@ -26,7 +26,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
-import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import spoon.Launcher;
@@ -38,42 +38,69 @@
 
 public class ModelStreamerTest {
 
-	private final String filename = "./src/test/resources/serialization/factory.ser";
+	private static final String SOURCE_DIRECTORY = "./src/main/java/spoon/reflect/declaration";
+	private final static String OUTPUT_FILENAME = "./src/test/resources/serialization/factory.ser";
+	private static File outputFile;
+	private static Factory factory;
 
-	private Factory factory;
-
-	private File file;
-
-	@Before
-    public void setUp() {
+	@BeforeClass
+    public static void setUp() {
+		System.out.println("Testing factory serialization with different "
+						 + "compressors with source folder: " + SOURCE_DIRECTORY);
 		Launcher launcher = new Launcher();
-		launcher.addInputResource("./src/main/java/spoon/reflect/declaration");
+		launcher.addInputResource(SOURCE_DIRECTORY);
 		launcher.buildModel();
-		this.factory = launcher.getFactory();
-		file = new File(filename);
-		file.deleteOnExit();
+		factory = launcher.getFactory();
+		outputFile = new File(OUTPUT_FILENAME);
+		outputFile.deleteOnExit();
     }
 
 	@Test
 	public void testDefaultCompressionType() throws IOException {
-		new SerializationModelStreamer().save(factory, new FileOutputStream(file));
-		Factory factoryFromFile = new SerializationModelStreamer().load(new FileInputStream(file));
+		new SerializationModelStreamer().save(factory, new FileOutputStream(outputFile));
+		FileInputStream in = new FileInputStream(outputFile);
+		System.out.println(in.getChannel().size() + " bytes for default");
+		Factory factoryFromFile = new SerializationModelStreamer().load(in);
 		compareFactoryModels(factory, factoryFromFile);
 	}
-	
+
 	@Test
 	public void testGZipCompressionType() throws IOException {
 		factory.getEnvironment().setCompressionType(CompressionType.GZIP);
-		new SerializationModelStreamer().save(factory, new FileOutputStream(file));
-		Factory factoryFromFile = new SerializationModelStreamer().load(new FileInputStream(file));
+		new SerializationModelStreamer().save(factory, new FileOutputStream(outputFile));
+		FileInputStream in = new FileInputStream(outputFile);
+		System.out.println(in.getChannel().size() + " bytes for " + CompressionType.GZIP);
+		Factory factoryFromFile = new SerializationModelStreamer().load(in);
+		compareFactoryModels(factory, factoryFromFile);
+	}
+
+	@Test
+	public void testBZip2CompressionType() throws IOException {
+		factory.getEnvironment().setCompressionType(CompressionType.BZIP2);
+		new SerializationModelStreamer().save(factory, new FileOutputStream(outputFile));
+		FileInputStream in = new FileInputStream(outputFile);
+		System.out.println(in.getChannel().size() + " bytes for " + CompressionType.BZIP2);
+		Factory factoryFromFile = new SerializationModelStreamer().load(in);
 		compareFactoryModels(factory, factoryFromFile);
 	}
 
 	@Test
 	public void testNoneCompressionType() throws IOException {
 		factory.getEnvironment().setCompressionType(CompressionType.NONE);
-		new SerializationModelStreamer().save(factory, new FileOutputStream(file));
-		Factory factoryFromFile = new SerializationModelStreamer().load(new FileInputStream(file));
+		new SerializationModelStreamer().save(factory, new FileOutputStream(outputFile));
+		FileInputStream in = new FileInputStream(outputFile);
+		System.out.println(in.getChannel().size() + " bytes for " + CompressionType.NONE);
+		Factory factoryFromFile = new SerializationModelStreamer().load(in);
+		compareFactoryModels(factory, factoryFromFile);
+	}
+
+	@Test
+	public void testLZMACompressionType() throws IOException {
+		factory.getEnvironment().setCompressionType(CompressionType.LZMA);
+		new SerializationModelStreamer().save(factory, new FileOutputStream(outputFile));
+		FileInputStream in = new FileInputStream(outputFile);
+		System.out.println(in.getChannel().size() + " bytes for " + CompressionType.LZMA);
+		Factory factoryFromFile = new SerializationModelStreamer().load(in);
 		compareFactoryModels(factory, factoryFromFile);
 	}