Skip to content

Feature: Parallel Building #3476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 26, 2023
Merged

Feature: Parallel Building #3476

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4f7291b
perf: process smali code in parallel
Cregrant Dec 19, 2023
390a8fd
perf: start backsmali concurrently with a resources decompiler
Cregrant Dec 19, 2023
f410812
perf: speed up apk building by skipping temp archive creation
Cregrant Dec 20, 2023
d9db843
refactor: extract duplicated code
Cregrant Dec 21, 2023
9650d82
refactor: rename methods and inline some comments
Cregrant Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 94 additions & 110 deletions brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,19 @@
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

public class ApkBuilder {
private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());

private final AtomicReference<AndrolibException> mBuildError = new AtomicReference<>(null);
private final Config mConfig;
private final ExtFile mApkDir;
private BackgroundWorker mWorker;
private ApkInfo mApkInfo;
private int mMinSdkVersion = 0;

Expand All @@ -78,48 +80,53 @@ public ApkBuilder(Config config, ExtFile apkDir) {

public void build(File outFile) throws BrutException {
LOGGER.info("Using Apktool " + ApktoolProperties.getVersion());
try {
mWorker = new BackgroundWorker();
mApkInfo = ApkInfo.load(mApkDir);

mApkInfo = ApkInfo.load(mApkDir);
if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}

if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
}
if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}

if (outFile == null) {
String outFileName = mApkInfo.apkFileName;
outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
}
//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

scheduleBuildSources(); //create classes.dex
scheduleBuildNonDefaultSources(); //create all other dex files
buildManifestFile(manifest, manifestOriginal); //backup AndroidManifest.xml
buildResources(); //create res folder, manifest file and resources.arsc
buildLibs(); //copy lib, libs, kotlin and META-INF/services
buildCopyOriginalFiles(); //copy some original files if they don't exist
mWorker.waitForFinish(); //wait until all previous build tasks are finished
if (mBuildError.get() != null) {
throw mBuildError.get();
}

//noinspection ResultOfMethodCallIgnored
new File(mApkDir, APK_DIRNAME).mkdirs();
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");

buildSources();
buildNonDefaultSources();
buildManifestFile(manifest, manifestOriginal);
buildResources();
buildLibs();
buildCopyOriginalFiles();
buildApk(outFile);

// we must go after the Apk is built, and copy the files in via Zip
// this is because Aapt won't add files it doesn't know (ex unknown files)
buildUnknownFiles(outFile);

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
buildApk(outFile); //finish the apk by adding common and unknown files

// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
try {
if (new File(mApkDir, "AndroidManifest.xml").delete()) {
FileUtils.moveFile(manifestOriginal, manifest);
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
} catch (IOException ex) {
throw new AndrolibException(ex.getMessage());
}
LOGGER.info("Built apk into: " + outFile.getPath());
} finally {
mWorker.shutdownNow();
}
LOGGER.info("Built apk into: " + outFile.getPath());
}

private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
Expand All @@ -141,13 +148,23 @@ private void buildManifestFile(File manifest, File manifestOriginal) throws Andr
}
}

private void buildSources() throws AndrolibException {
if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
LOGGER.warning("Could not find sources");
}
private void scheduleBuildSources() {
Runnable r = () -> {
try {
if (mBuildError.get() != null) {
return;
}
if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
LOGGER.warning("Could not find sources");
}
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, e);
}
};
mWorker.submit(r);
}

private void buildNonDefaultSources() throws AndrolibException {
private void scheduleBuildNonDefaultSources() throws AndrolibException {
try {
// loop through any smali_ directories for multi-dex apks
Map<String, Directory> dirs = mApkDir.getDirectory().getDirs();
Expand All @@ -156,9 +173,19 @@ private void buildNonDefaultSources() throws AndrolibException {
if (name.startsWith("smali_")) {
String filename = name.substring(name.indexOf("_") + 1) + ".dex";

if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
LOGGER.warning("Could not find sources");
}
Runnable r = () -> {
try {
if (mBuildError.get() != null) {
return;
}
if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
LOGGER.warning("Could not find sources");
}
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, e);
}
};
mWorker.submit(r);
}
}

Expand Down Expand Up @@ -427,49 +454,33 @@ private void buildCopyOriginalFiles() throws AndrolibException {
}
}

private void buildUnknownFiles(File outFile) throws AndrolibException {
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");

Map<String, String> files = mApkInfo.unknownFiles;
File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
boolean renamed = outFile.renameTo(tempFile);
if (!renamed) {
throw new AndrolibException("Unable to rename temporary file");
}

try (
ZipFile inputFile = new ZipFile(tempFile);
ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
) {
copyExistingFiles(inputFile, actualOutput);
copyUnknownFiles(actualOutput, files);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}

// Remove our temporary file.
private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
}

private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
// First, copy the contents from the existing outFile:
Enumeration<? extends ZipEntry> entries = inputFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = new ZipEntry(entries.nextElement());

// We can't reuse the compressed size because it depends on compression sizes.
entry.setCompressedSize(-1);
outputFile.putNextEntry(entry);
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) {
ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress);

// No need to create directory entries in the final apk
if (!entry.isDirectory()) {
BrutIO.copy(inputFile, outputFile, entry);
// we must go after the Apk is built, and copy the files in via Zip
// this is because Aapt won't add files it doesn't know (ex unknown files)
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");
copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles);
}

outputFile.closeEntry();
} catch (IOException | BrutException e) {
throw new AndrolibException(e);
}
}

Expand Down Expand Up @@ -513,33 +524,6 @@ private void copyUnknownFiles(ZipOutputStream outputFile, Map<String, String> fi
}
}

private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir);
}

private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
try {
ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}
}

private File[] getIncludeFiles() throws AndrolibException {
UsesFramework usesFramework = mApkInfo.usesFramework;
if (usesFramework == null) {
Expand Down
Loading