Skip to content

perf: Gzip model serialization #2103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jun 28, 2018
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/main/java/spoon/compiler/Environment.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import spoon.reflect.declaration.CtElement;
import spoon.reflect.declaration.CtMethod;
import spoon.support.OutputDestinationHandler;
import spoon.support.CompressionType;
import spoon.support.compiler.SpoonProgress;

import java.io.File;
Expand Down Expand Up @@ -402,4 +403,14 @@ void report(Processor<?> processor, Level level,
SpoonProgress getSpoonProgress();

void setSpoonProgress(SpoonProgress spoonProgress);

/**
* Get the type of serialization to be used by default
*/
CompressionType getCompressionType();

/**
* Set the type of serialization to be used by default
*/
void setCompressionType(CompressionType serializationType);
}
4 changes: 4 additions & 0 deletions src/main/java/spoon/reflect/ModelStreamer.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
import java.io.InputStream;
import java.io.OutputStream;

import spoon.compiler.Environment;
import spoon.reflect.factory.Factory;
import spoon.support.CompressionType;

/**
* This interface defines the protocol to save and load a factory and it's
Expand All @@ -30,6 +32,7 @@ public interface ModelStreamer {

/**
* Saves a factory (and all its associated Java program elements).
* Stream is GZIP compressed by default, see {@link Environment#setCompressionType(spoon.support.CompressionType)}
*
* @param f
* the factory to be save
Expand All @@ -42,6 +45,7 @@ public interface ModelStreamer {

/**
* Loads a factory (and all its associated Java program elements).
* Tries to decompress the file given the available {@link CompressionType}
*
* @param in
* the used input stream
Expand Down
28 changes: 28 additions & 0 deletions src/main/java/spoon/support/CompressionType.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/**
* Copyright (C) 2006-2018 INRIA and contributors
* Spoon - http://spoon.gforge.inria.fr/
*
* This software is governed by the CeCILL-C License under French law and
* abiding by the rules of distribution of free software. You can use, modify
* and/or redistribute the software under the terms of the CeCILL-C license as
* circulated by CEA, CNRS and INRIA at http://www.cecill.info.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the CeCILL-C License for more details.
*
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*/
package spoon.support;

/**
* Different types of compressions used, e.g. for serialization
* @see SerializationModelStreamer
*/
public enum CompressionType {

NONE,
GZIP;

}
36 changes: 29 additions & 7 deletions src/main/java/spoon/support/SerializationModelStreamer.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@
*/
package spoon.support;

import spoon.Launcher;
import spoon.reflect.ModelStreamer;
import spoon.reflect.declaration.CtElement;
import spoon.reflect.factory.Factory;
import spoon.reflect.visitor.Filter;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import spoon.Launcher;
import spoon.reflect.ModelStreamer;
import spoon.reflect.declaration.CtElement;
import spoon.reflect.factory.Factory;
import spoon.reflect.visitor.Filter;

/**
* This class provides a regular Java serialization-based implementation of the
Expand All @@ -43,6 +45,9 @@ public SerializationModelStreamer() {
}

public void save(Factory f, OutputStream out) throws IOException {
if (f.getEnvironment().getCompressionType() == CompressionType.GZIP) {
out = new GZIPOutputStream(out);
}
ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(out));
oos.writeObject(f);
oos.flush();
Expand All @@ -51,7 +56,24 @@ public void save(Factory f, OutputStream out) throws IOException {

public Factory load(InputStream in) throws IOException {
try {
ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(in));

BufferedInputStream buffered = new BufferedInputStream(in, 2);

// Check if its a GZIP
buffered.mark(2);
int[] buffer = new int[2];
buffer[0] = buffered.read();
buffer[1] = buffered.read();
buffered.reset();

int header = (buffer[1] << 8) | buffer[0];
if (header == GZIPInputStream.GZIP_MAGIC) {
in = new GZIPInputStream(buffered);
} else {
in = buffered;
}

ObjectInputStream ois = new ObjectInputStream(in);
final Factory f = (Factory) ois.readObject();
//create query using factory directly
//because any try to call CtElement#map or CtElement#filterChildren will fail on uninitialized factory
Expand Down
12 changes: 12 additions & 0 deletions src/main/java/spoon/support/StandardEnvironment.java
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ public class StandardEnvironment implements Serializable, Environment {

private SpoonProgress spoonProgress = null;

private CompressionType compressionType = CompressionType.GZIP;

/**
* Creates a new environment with a <code>null</code> default file
* generator.
Expand Down Expand Up @@ -588,4 +590,14 @@ public SpoonProgress getSpoonProgress() {
public void setSpoonProgress(SpoonProgress spoonProgress) {
this.spoonProgress = spoonProgress;
}

@Override
public CompressionType getCompressionType() {
return compressionType;
}

@Override
public void setCompressionType(CompressionType serializationType) {
this.compressionType = serializationType;
}
}
80 changes: 80 additions & 0 deletions src/test/java/spoon/test/serializable/ModelStreamerTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package spoon.test.serializable;

import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import org.junit.Before;
import org.junit.Test;

import spoon.Launcher;
import spoon.reflect.declaration.CtElement;
import spoon.reflect.factory.Factory;
import spoon.reflect.visitor.Filter;
import spoon.support.CompressionType;
import spoon.support.SerializationModelStreamer;

public class ModelStreamerTest {

private final String filename = "./src/test/resources/serialization/factory.ser";

private Factory factory;

private File file;

@Before
public void setUp() {
Launcher launcher = new Launcher();
launcher.addInputResource("./src/main/java/spoon/reflect/declaration");
launcher.buildModel();
this.factory = launcher.getFactory();
file = new File(filename);
file.deleteOnExit();
}

@Test
public void testGZipCompression() throws IOException {

Copy link
Collaborator

@surli surli Jun 27, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest to specify explicitely that you're using GZIP compression type (e.g. to set it in the environment): the default value might evolve in the future.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right! I'll do that tonight after work.

new SerializationModelStreamer().save(factory, new FileOutputStream(file));
Factory factoryFromFile = new SerializationModelStreamer().load(new FileInputStream(file));
compareFactoryModels(factory, factoryFromFile);
}

@Test
public void testNoCompression() throws IOException {
factory.getEnvironment().setCompressionType(CompressionType.NONE);
new SerializationModelStreamer().save(factory, new FileOutputStream(file));
Factory factoryFromFile = new SerializationModelStreamer().load(new FileInputStream(file));
compareFactoryModels(factory, factoryFromFile);
}

private void compareFactoryModels(Factory factory, Factory factoryFromFile) {
Filter<CtElement> filter = new Filter<CtElement>() {
public boolean matches(CtElement element) {
return true;
};
};

List<CtElement> elementsFactory = factory.getModel().getRootPackage().filterChildren(filter).list();
List<CtElement> elementsFactoryFromFile = factoryFromFile.getModel().getRootPackage().filterChildren(filter).list();

assertTrue("Model before & after serialization must have the same number of elements",
elementsFactory.size() == elementsFactoryFromFile.size());

List<String> st1 = elementsFactory.stream().map(CtElement::toString).collect(Collectors.toList());
List<String> st2 = elementsFactoryFromFile.stream().map(CtElement::toString).collect(Collectors.toList());
Collections.sort(st1);
Collections.sort(st2);

for (int i = 0; i < st1.size(); i++) {
assertTrue("All CtElement of the model should be striclty identical before & after serialization",
st1.get(i).equals(st2.get(i)));
}
}
}