From 94d252b854ef1eef98604f35bb87c7e7694c6e1c Mon Sep 17 00:00:00 2001 From: yangzhiqian Date: Mon, 15 Jun 2020 20:34:01 +0800 Subject: [PATCH 1/2] =?UTF-8?q?merge=20master:Upgrade=20AGP(3.5.3)?= =?UTF-8?q?=E3=80=81Fix=20memory=20leak=E3=80=81Run=20refer-check=20in=20s?= =?UTF-8?q?ingle=E3=80=81Unified=20threadpool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- GradleToolKit/GradleEnvApi/build.gradle | 2 +- GradleToolKit/build.gradle | 6 +-- .../ugc/bytex/gradletoolkit/MergeResources.kt | 41 ++++++++++++++----- .../gradletoolkit/MergeSourceSetFolders.kt | 37 +++++++++++------ .../KeepClassSpecificationHolder.java | 6 +-- .../PluginConfigProcessor/build.gradle | 1 + TransformEngine/build.gradle | 2 +- .../bytex/transformer/TransformContext.java | 20 ++++++++- .../bytex/transformer/TransformEngine.java | 20 +++++++-- .../ugc/bytex/transformer/TransformInputs.kt | 17 +++++++- .../ugc/bytex/transformer/cache/DirCache.java | 14 +++++-- .../bytex/transformer/cache/FileCache.java | 12 +++++- .../ugc/bytex/transformer/cache/JarCache.java | 12 +++++- .../bytex/transformer/cache/NewFileCache.java | 13 +++--- .../bytex/transformer/concurrent/Worker.java | 26 +++++++++--- build.gradle | 2 +- common/build.gradle | 3 +- .../android/ugc/bytex/common/AbsPlugin.java | 6 ++- .../ugc/bytex/common/CommonTransform.java | 4 +- .../bytex/common/flow/AbsTransformFlow.java | 4 +- .../common/flow/main/MainTransformFlow.java | 15 ++++--- .../common/graph/cache/GsonGraphCache.kt | 8 +++- .../bytex/common/graph/cache/RAMGraphCache.kt | 8 +++- gradle.properties | 1 + gradle/ext.gradle | 5 ++- gradle/plugin.gradle | 2 +- gradle/publish.gradle | 5 ++- gradle/wrapper/gradle-wrapper.properties | 2 +- publish.sh | 1 + .../bytex/refercheck/ReferCheckPlugin.java | 13 ++++++ wiki/ByteX-Developer-API-en.md | 2 +- wiki/ByteX-Developer-API-zh.md | 2 +- 32 files changed, 232 insertions(+), 80 deletions(-) diff --git a/GradleToolKit/GradleEnvApi/build.gradle b/GradleToolKit/GradleEnvApi/build.gradle index 436da56..6caec82 100644 --- a/GradleToolKit/GradleEnvApi/build.gradle +++ b/GradleToolKit/GradleEnvApi/build.gradle @@ -3,7 +3,7 @@ group "$upload_group" version "$upload_version" dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - compile gradleApi() + compileOnly gradleApi() compileOnly "com.android.tools.build:gradle:$gradle_version" } sourceCompatibility = "1.8" diff --git a/GradleToolKit/build.gradle b/GradleToolKit/build.gradle index 1884421..6de6b0c 100644 --- a/GradleToolKit/build.gradle +++ b/GradleToolKit/build.gradle @@ -6,15 +6,15 @@ group "$upload_group" version "$upload_version" dependencies { kapt 'com.google.auto.service:auto-service:1.0-rc4' - compile gradleApi() + compileOnly gradleApi() compileOnly "com.android.tools.build:gradle:$gradle_version" compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" compile project(':GradleEnvApi') - compile ('com.google.auto.service:auto-service:1.0-rc4') { + compile('com.google.auto.service:auto-service:1.0-rc4') { exclude module: 'guava' } - implementation "com.didiglobal.booster:booster-android-gradle-api:1.2.0" + implementation "com.didiglobal.booster:booster-android-gradle-api:2.0.0" } sourceCompatibility = "1.8" targetCompatibility = "1.8" diff --git a/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeResources.kt b/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeResources.kt index 7ef4919..dee6f63 100644 --- a/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeResources.kt +++ b/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeResources.kt @@ -5,29 +5,48 @@ package com.ss.android.ugc.bytex.gradletoolkit */ import com.android.build.gradle.tasks.MergeResources +import com.android.ide.common.resources.ResourceSet import java.io.File fun MergeResources.resourceSetList(): List { + val resourceSets = try { + resourceSetList1() + } catch (e: Exception) { + resourceSetList2() + } + return resourceSets.flatMap { it.sourceFiles }.toSet().toList() +} + + +fun MergeResources.resourceSetList1(): Iterable { val computeResourceSetListMethod = MergeResources::class.java.declaredMethods - .firstOrNull { it.name == "computeResourceSetList" && it.parameterCount == 0 } - ?: return emptyList() + .find { it.name == "computeResourceSetList" && it.parameterCount == 0 }!! val oldIsAccessible = computeResourceSetListMethod.isAccessible try { computeResourceSetListMethod.isAccessible = true + return computeResourceSetListMethod.invoke(this) as Iterable + } finally { + computeResourceSetListMethod.isAccessible = oldIsAccessible + } +} - val resourceSets = computeResourceSetListMethod.invoke(this) as? Iterable<*> - ?: return emptyList() +fun MergeResources.resourceSetList2(): Iterable { + val getConfiguredResourceSets = MergeResources::class.java.declaredMethods + .find { it.name == "getConfiguredResourceSets" && it.parameterCount == 1 }!! - return resourceSets.mapNotNull { resourceSet -> - val getSourceFiles = resourceSet?.javaClass?.methods?.find { it.name == "getSourceFiles" && it.parameterCount == 0 } - val files = getSourceFiles?.invoke(resourceSet) - @Suppress("UNCHECKED_CAST") - files as? Iterable - }.flatten() + val getPreprocessor = MergeResources::class.java.declaredMethods + .find { it.name == "getPreprocessor" && it.parameterCount == 0 }!! + val getConfiguredResourceSetsAccess = getConfiguredResourceSets.isAccessible + val getPreprocessorAccess = getPreprocessor.isAccessible + try { + getConfiguredResourceSets.isAccessible = true + getPreprocessor.isAccessible = true + return getConfiguredResourceSets.invoke(this, getPreprocessor.invoke(this)) as Iterable } finally { - computeResourceSetListMethod.isAccessible = oldIsAccessible + getConfiguredResourceSets.isAccessible = getConfiguredResourceSetsAccess + getPreprocessor.isAccessible = getPreprocessorAccess } } \ No newline at end of file diff --git a/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeSourceSetFolders.kt b/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeSourceSetFolders.kt index e770022..9a4996f 100644 --- a/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeSourceSetFolders.kt +++ b/GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeSourceSetFolders.kt @@ -1,6 +1,7 @@ package com.ss.android.ugc.bytex.gradletoolkit import com.android.build.gradle.tasks.MergeSourceSetFolders +import com.android.ide.common.resources.AssetSet import java.io.File /** @@ -9,27 +10,37 @@ import java.io.File fun MergeSourceSetFolders.assetSetList(): List { + val assetSets = try { + assetSetList1() + } catch (e: Exception) { + assetSetList2() + } + return assetSets.flatMap { it.sourceFiles }.toSet().toList() +} + +fun MergeSourceSetFolders.assetSetList1(): Iterable { val computeAssetSetListMethod = MergeSourceSetFolders::class.java.declaredMethods - .firstOrNull { it.name == "computeAssetSetList" && it.parameterCount == 0 } - ?: return emptyList() + .find { it.name == "computeAssetSetList" && it.parameterCount == 0 }!! val oldIsAccessible = computeAssetSetListMethod.isAccessible try { computeAssetSetListMethod.isAccessible = true + return computeAssetSetListMethod.invoke(this) as Iterable + } finally { + computeAssetSetListMethod.isAccessible = oldIsAccessible + } +} - val assetSets = computeAssetSetListMethod.invoke(this) as? Iterable<*> - ?: return emptyList() - - return assetSets.mapNotNull { assetSet -> - val getSourceFiles = assetSet?.javaClass?.methods?.find { it.name == "getSourceFiles" && it.parameterCount == 0 } - val files = getSourceFiles?.invoke(assetSet) - @Suppress("UNCHECKED_CAST") - files as? Iterable - }.flatten() +fun MergeSourceSetFolders.assetSetList2(): Iterable { + val computeAssetSetListMethod = MergeSourceSetFolders::class.java.declaredMethods + .find { it.name == "computeAssetSetList\$gradle" && it.parameterCount == 0 }!! + val oldIsAccessible = computeAssetSetListMethod.isAccessible + try { + computeAssetSetListMethod.isAccessible = true + return computeAssetSetListMethod.invoke(this) as Iterable } finally { computeAssetSetListMethod.isAccessible = oldIsAccessible } - -} \ No newline at end of file +} diff --git a/HookProguard/src/main/java/com/ss/android/ugc/bytex/hookproguard/KeepClassSpecificationHolder.java b/HookProguard/src/main/java/com/ss/android/ugc/bytex/hookproguard/KeepClassSpecificationHolder.java index 5b16108..e7fba63 100644 --- a/HookProguard/src/main/java/com/ss/android/ugc/bytex/hookproguard/KeepClassSpecificationHolder.java +++ b/HookProguard/src/main/java/com/ss/android/ugc/bytex/hookproguard/KeepClassSpecificationHolder.java @@ -15,7 +15,6 @@ class KeepClassSpecificationHolder { private final KeepClassSpecification instance; private StringMatcher classNameMatcher; private List methodSpecifications; - private Node extendsClassNode; KeepClassSpecificationHolder(KeepClassSpecification instance, StringParser parser) { this.instance = instance; @@ -50,9 +49,6 @@ void parserMethodSpecifications(StringParser parser) { } Node computeExtendsClassNode(Graph graph) { - if (extendsClassNode == null) { - extendsClassNode = graph.get(instance.extendsClassName); - } - return extendsClassNode; + return graph.get(instance.extendsClassName); } } diff --git a/PluginConfig/PluginConfigProcessor/build.gradle b/PluginConfig/PluginConfigProcessor/build.gradle index d37cba5..588d28b 100644 --- a/PluginConfig/PluginConfigProcessor/build.gradle +++ b/PluginConfig/PluginConfigProcessor/build.gradle @@ -3,6 +3,7 @@ apply plugin: 'java' dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) compile('com.google.auto.service:auto-service:1.0-rc4') + annotationProcessor 'com.google.auto.service:auto-service:1.0-rc4' compile project(':PluginConfigAnnotation') } sourceCompatibility = "1.8" diff --git a/TransformEngine/build.gradle b/TransformEngine/build.gradle index 34d5b30..520603d 100644 --- a/TransformEngine/build.gradle +++ b/TransformEngine/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'java-library' apply plugin: 'kotlin' dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - compile gradleApi() + compileOnly gradleApi() compileOnly "com.android.tools.build:gradle:$gradle_version" compile "com.google.guava:guava:$guava_version" testCompile 'com.squareup.okio:okio:1.14.0' diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformContext.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformContext.java index 34213c6..3b27da4 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformContext.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformContext.java @@ -120,7 +120,8 @@ public boolean shouldSaveCache() { } /** - * 请求非增量运行，必须在traverse时机之前调用 + * 请求非增量运行，必须在traverse时机之前调用
+ * beforeTraverse及之前生命周期可调用，否则报RuntimeException
*/ public void requestNotIncremental() { if (running.get()) { @@ -130,6 +131,23 @@ public void requestNotIncremental() { this.transformInputs.requestNotIncremental(); } + /** + * 请求某个文件进行非增量
+ * beforeTraverse及之前生命周期可调用，否则报RuntimeException
+ * + * @param relativePath 文件的相对路径,比如 com/bytedance/Demo.class + * @return 成功修改对应输入的状态，如果当前已经是非增量 + */ + public boolean requestNotIncremental(String relativePath) { + if (running.get()) { + throw new RuntimeException("You Should request for not incremental before traversing."); + } + if (!this.isIncremental()) { + return false; + } + return this.transformInputs.requestNotIncremental(relativePath); + } + public boolean isReleaseBuild() { return invocation.getContext().getVariantName().toLowerCase().contains("release"); } diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformEngine.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformEngine.java index 68d56f2..d262a7c 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformEngine.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformEngine.java @@ -28,11 +28,12 @@ public class TransformEngine { public TransformEngine(TransformContext context) { this.context = context; } - public void beginRun(){ + + public void beginRun() { context.markRunningState(false); } - public void running(){ + public void running() { context.markRunningState(true); } @@ -55,6 +56,18 @@ public void skip() throws IOException { worker.await(); } + public void transformOutput() throws IOException { + Worker worker = Schedulers.IO(); + context.allFiles() + .filter(fileCache -> !fileCache.isHasWritten()) + .map(f -> (Callable) () -> { + f.transformOutput(); + return null; + }) + .forEach(worker::submit); + worker.await(); + } + public void traverseOnly(FileProcessor... processors) { Schedulers.FORKJOINPOOL().invoke(new PerformTraverseTask(context.allFiles(), getProcessorList(processors))); } @@ -70,9 +83,10 @@ private static List getProcessorList(FileProcessor[] processors) return realProcessorList; } - public void endRun(){ + public void endRun() { } + public TransformContext getContext() { return context; } diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformInputs.kt b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformInputs.kt index a5b9c17..9a16a46 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformInputs.kt +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformInputs.kt @@ -116,9 +116,22 @@ class TransformInputs internal constructor(private val context: TransformContext } protected fun requestNotIncremental() { - transformInputs.flatMap { it.value }.parallelStream().forEach { - it.status = Status.ADDED + transformInputs.flatMap { it.value }.forEach { + if (it.status == Status.NOTCHANGED) { + it.status = Status.CHANGED + } + } + } + + protected fun requestNotIncremental(relativePath: String): Boolean { + var r = false + transformInputs.flatMap { it.value }.forEach { + if (it.relativePath == relativePath && it.status == Status.NOTCHANGED) { + it.status = Status.CHANGED + r = true + } } + return r } fun addFile(affinity: String, file: FileData) { diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/DirCache.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/DirCache.java index 94743ad..a0dc179 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/DirCache.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/DirCache.java @@ -50,7 +50,10 @@ public File getFile() { } @Override - public void transformOutput(Consumer visitor) throws IOException { + public final synchronized void transformOutput(Consumer visitor) throws IOException { + if (hasWritten) { + throw new RuntimeException("rewrite"); + } Map entryMap = new HashMap<>(); String relativeToProject = context.getTransformOutputs().relativeToProject(outputFile); TransformOutputs.Entry entry = context.getTransformOutputs().getLastTransformOutputs().get(relativeToProject); @@ -59,7 +62,7 @@ public void transformOutput(Consumer visitor) throws IOException { entry.traverseAll(e -> entryMap.put(e.getPath(), e)); } List entries = Collections.synchronizedList(new LinkedList<>()); - parallelForEach(true, item -> { + parallelForEach(false, item -> { if (visitor != null) visitor.accept(item); entries.add( transformOutput( @@ -76,6 +79,7 @@ public void transformOutput(Consumer visitor) throws IOException { 0L, Collections.unmodifiableList(entries)) ); + hasWritten = true; } private TransformOutputs.Entry transformOutput(String input, String parent, FileData fileData, Map entryMap) throws IOException { @@ -183,7 +187,11 @@ protected List resolve(ObservableEmitter emitter) throws IOE } @Override - public void skip() throws IOException { + public synchronized void skip() throws IOException { + if (hasWritten) { + throw new RuntimeException("rewrite"); + } FileUtils.copyDirectory(getFile(), outputFile); + hasWritten = true; } } diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/FileCache.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/FileCache.java index c63ce82..dac8261 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/FileCache.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/FileCache.java @@ -3,6 +3,7 @@ import com.android.build.api.transform.QualifiedContent; import com.android.build.api.transform.Status; import com.ss.android.ugc.bytex.transformer.TransformContext; +import com.ss.android.ugc.bytex.transformer.concurrent.Schedulers; import java.io.File; import java.io.IOException; @@ -18,6 +19,8 @@ public abstract class FileCache implements Serializable { protected QualifiedContent content; protected TransformContext context; protected List files; + protected boolean hasRead = false; + protected boolean hasWritten = false; public FileCache(QualifiedContent content, TransformContext context) { this.content = content; @@ -44,9 +47,10 @@ public final Observable stream(boolean tryParallel) { synchronized (this) { if (files == null) { files = resolve(emitter); + hasRead = true; } else { if (tryParallel) { - files.parallelStream().forEach(emitter::onNext); + Schedulers.IO().submitAndAwait(files, emitter::onNext); } else { files.forEach(emitter::onNext); } @@ -54,7 +58,7 @@ public final Observable stream(boolean tryParallel) { } } else { if (tryParallel) { - files.parallelStream().forEach(emitter::onNext); + Schedulers.IO().submitAndAwait(files, emitter::onNext); } else { files.forEach(emitter::onNext); } @@ -93,4 +97,8 @@ public File getFile() { public boolean containsFileData(String relativePath) { return stream().filter(fileData -> fileData.getStatus() != Status.REMOVED && fileData.getRelativePath().equals(relativePath)).firstElement().blockingGet() != null; } + + public boolean isHasWritten() { + return hasWritten; + } } diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/JarCache.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/JarCache.java index 5b1eb0b..5091cd6 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/JarCache.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/JarCache.java @@ -58,7 +58,10 @@ public JarCache(File jar, Status status, TransformContext context) { } @Override - public void transformOutput(Consumer visitor) throws IOException { + public final synchronized void transformOutput(Consumer visitor) throws IOException { + if (hasWritten) { + throw new RuntimeException("rewrite"); + } List dataList = Collections.synchronizedList(new LinkedList<>()); AtomicBoolean needOutput = new AtomicBoolean(!context.getInvocation().isIncremental()); forEach(item -> { @@ -117,6 +120,7 @@ public void transformOutput(Consumer visitor) throws IOException { } context.getTransformOutputs().getTransformOutputs().put(outputRelativePath, outputs); + hasWritten = true; } @Override @@ -209,8 +213,12 @@ protected List resolve(ObservableEmitter emitter) throws IOE } @Override - public void skip() throws IOException { + public synchronized void skip() throws IOException { + if (hasWritten) { + throw new RuntimeException("rewrite"); + } FileUtils.copyFile(getFile(), outputFile); + hasWritten = true; } @Override diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/NewFileCache.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/NewFileCache.java index 3534cd0..cc275ed 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/NewFileCache.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/NewFileCache.java @@ -11,7 +11,6 @@ import io.reactivex.ObservableEmitter; public class NewFileCache extends DirCache { - private final List newFiles; private final String affinity; public NewFileCache(TransformContext context, String affinity) { @@ -20,13 +19,14 @@ public NewFileCache(TransformContext context, String affinity) { public NewFileCache(List newFiles, String affinity, TransformContext context) { super(new File(affinity), getOutput(context, affinity), context); - this.newFiles = Collections.synchronizedList(newFiles); + this.files = Collections.synchronizedList(newFiles); + hasRead = true; this.affinity = affinity; } @Override protected List resolve(ObservableEmitter emitter) throws IOException { - return Collections.unmodifiableList(newFiles); + return Collections.unmodifiableList(files); } @Override @@ -37,14 +37,17 @@ public void skip() throws IOException { @Override public List getChangedFiles() { if (context.isIncremental()) { - return Collections.unmodifiableList(newFiles); + return Collections.unmodifiableList(files); } else { return Collections.emptyList(); } } public void addFile(FileData file) { - newFiles.add(file); + if (hasWritten) { + throw new RuntimeException("can not add file after FileCache has been outputed"); + } + files.add(file); } @Override diff --git a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/concurrent/Worker.java b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/concurrent/Worker.java index efa79f6..b3d653c 100644 --- a/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/concurrent/Worker.java +++ b/TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/concurrent/Worker.java @@ -1,15 +1,26 @@ package com.ss.android.ugc.bytex.transformer.concurrent; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.Collection; +import java.util.LinkedList; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Consumer; public class Worker { - protected final List> futures = new ArrayList<>(); + protected final LinkedList> futures = new LinkedList>() { + @Override + public synchronized boolean add(Future future) { + return super.add(future); + } + + @Override + public synchronized Future pollFirst() { + return super.pollFirst(); + } + }; protected ExecutorService executor; Worker(ExecutorService executor) { @@ -27,7 +38,8 @@ public Future submit(Callable callable) { } public void await() throws IOException { - for (Future future : futures) { + Future future; + while ((future = futures.pollFirst()) != null) { try { future.get(); } catch (ExecutionException | InterruptedException e) { @@ -41,6 +53,10 @@ public void await() throws IOException { throw new RuntimeException(e.getCause()); } } - futures.clear(); + } + + public void submitAndAwait(Collection is, Consumer consumer) throws IOException { + is.stream().map(f -> (Runnable) () -> consumer.accept(f)).forEach(this::execute); + await(); } } diff --git a/build.gradle b/build.gradle index 756f305..fa02e5f 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ buildscript { } } dependencies { - classpath 'com.android.tools.build:gradle:3.2.1' + classpath "com.android.tools.build:gradle:$gradle_version" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/common/build.gradle b/common/build.gradle index 942ca7a..5c9f3b2 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'java-library' apply plugin: 'kotlin' dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - compile gradleApi() + compileOnly gradleApi() compileOnly "com.android.tools.build:gradle:$gradle_version" compile "com.google.guava:guava:$guava_version" compile "org.ow2.asm:asm:$asm_version" @@ -12,6 +12,7 @@ dependencies { compile "org.ow2.asm:asm-commons:$asm_version" compile project(':TransformEngine') compile project(':GradleToolKit') + compile "com.google.code.gson:gson:2.8.2" compile group: 'dom4j', name: 'dom4j', version: '1.6.1' } diff --git a/common/src/main/java/com/ss/android/ugc/bytex/common/AbsPlugin.java b/common/src/main/java/com/ss/android/ugc/bytex/common/AbsPlugin.java index dee9f11..cfcac2c 100644 --- a/common/src/main/java/com/ss/android/ugc/bytex/common/AbsPlugin.java +++ b/common/src/main/java/com/ss/android/ugc/bytex/common/AbsPlugin.java @@ -13,6 +13,7 @@ import org.gradle.api.UnknownDomainObjectException; import org.gradle.internal.reflect.Instantiator; import org.gradle.invocation.DefaultGradle; +import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; @@ -44,7 +45,10 @@ public boolean shouldSaveCache() { } @Override - public final void apply(Project project) { + public final void apply(@NotNull Project project) { + if(!transformConfiguration().isIncremental()){ + System.err.println("[ByteX Warning]:"+this.getClass().getName()+" does not yet support incremental build"); + } this.project = project; this.android = project.getExtensions().getByType(AppExtension.class); ProjectOptions.INSTANCE.init(project); diff --git a/common/src/main/java/com/ss/android/ugc/bytex/common/CommonTransform.java b/common/src/main/java/com/ss/android/ugc/bytex/common/CommonTransform.java index a97c652..ac45d90 100644 --- a/common/src/main/java/com/ss/android/ugc/bytex/common/CommonTransform.java +++ b/common/src/main/java/com/ss/android/ugc/bytex/common/CommonTransform.java @@ -176,7 +176,7 @@ public boolean shouldSaveCache() { @Override public final void transform(TransformInvocation transformInvocation) throws TransformException, InterruptedException, IOException { super.transform(transformInvocation); - if (!transformInvocation.isIncremental()) { + if (!transformInvocation.isIncremental() && transformInvocation.getOutputProvider() != null) { transformInvocation.getOutputProvider().deleteAll(); } TransformContext transformContext = getTransformContext(transformInvocation); @@ -212,6 +212,8 @@ public final void transform(TransformInvocation transformInvocation) throws Tran } } } + //对某些后期新增但未输出的文件进行输出 + transformEngine.transformOutput(); } else { transformEngine.skip(); } diff --git a/common/src/main/java/com/ss/android/ugc/bytex/common/flow/AbsTransformFlow.java b/common/src/main/java/com/ss/android/ugc/bytex/common/flow/AbsTransformFlow.java index 6eabc96..925a791 100644 --- a/common/src/main/java/com/ss/android/ugc/bytex/common/flow/AbsTransformFlow.java +++ b/common/src/main/java/com/ss/android/ugc/bytex/common/flow/AbsTransformFlow.java @@ -47,9 +47,9 @@ protected AbsTransformFlow transform(FileProcessor... processors) throws IOExcep return this; } - protected abstract AbsTransformFlow beforeTransform(TransformEngine transformEngine); + protected abstract AbsTransformFlow beforeTransform(TransformEngine transformEngine) throws IOException; - protected abstract AbsTransformFlow afterTransform(TransformEngine transformEngine); + protected abstract AbsTransformFlow afterTransform(TransformEngine transformEngine) throws IOException; public void endRun() { transformEngine.endRun(); diff --git a/common/src/main/java/com/ss/android/ugc/bytex/common/flow/main/MainTransformFlow.java b/common/src/main/java/com/ss/android/ugc/bytex/common/flow/main/MainTransformFlow.java index f410d8e..e0d66d4 100644 --- a/common/src/main/java/com/ss/android/ugc/bytex/common/flow/main/MainTransformFlow.java +++ b/common/src/main/java/com/ss/android/ugc/bytex/common/flow/main/MainTransformFlow.java @@ -10,6 +10,7 @@ import com.ss.android.ugc.bytex.common.processor.ClassFileAnalyzer; import com.ss.android.ugc.bytex.common.processor.ClassFileTransformer; import com.ss.android.ugc.bytex.transformer.TransformEngine; +import com.ss.android.ugc.bytex.transformer.concurrent.Schedulers; import com.ss.android.ugc.bytex.transformer.processor.ClassFileProcessor; import com.ss.android.ugc.bytex.transformer.processor.FileHandler; import com.ss.android.ugc.bytex.transformer.processor.FileProcessor; @@ -48,9 +49,7 @@ private void runTransform() throws IOException, InterruptedException { Timer timer = new Timer(); timer.startRecord("PRE_PROCESS"); timer.startRecord("INIT"); - for (MainProcessHandler handler : handlers) { - handler.init(transformEngine); - } + Schedulers.COMPUTATION().submitAndAwait(handlers, handler -> handler.init(transformEngine)); timer.stopRecord("INIT", "Process init cost time = [%s ms]"); if (!isOnePassEnough()) { if (!handlers.isEmpty() && context.isIncremental()) { @@ -59,7 +58,7 @@ private void runTransform() throws IOException, InterruptedException { timer.stopRecord("TRAVERSE_INCREMENTAL", "Process project all .class files cost time = [%s ms]"); } - handlers.forEach(plugin -> plugin.beforeTraverse(transformEngine)); + Schedulers.COMPUTATION().submitAndAwait(handlers, plugin -> plugin.beforeTraverse(transformEngine)); timer.startRecord("LOADCACHE"); GraphBuilder graphBuilder = new CachedGraphBuilder(context.getGraphCache(), context.isIncremental(), context.shouldSaveCache()); if (context.isIncremental() && !graphBuilder.isCacheValid()) { @@ -141,14 +140,14 @@ public final TransformFlow appendHandler(MainProcessHandler handler) { } @Override - protected AbsTransformFlow beforeTransform(TransformEngine transformEngine) { - handlers.forEach(plugin -> plugin.beforeTransform(transformEngine)); + protected AbsTransformFlow beforeTransform(TransformEngine transformEngine) throws IOException { + Schedulers.COMPUTATION().submitAndAwait(handlers, plugin -> plugin.beforeTransform(transformEngine)); return this; } @Override - protected AbsTransformFlow afterTransform(TransformEngine transformEngine) { - handlers.forEach(plugin -> plugin.afterTransform(transformEngine)); + protected AbsTransformFlow afterTransform(TransformEngine transformEngine) throws IOException { + Schedulers.COMPUTATION().submitAndAwait(handlers, plugin -> plugin.afterTransform(transformEngine)); return this; } diff --git a/common/src/main/java/com/ss/android/ugc/bytex/common/graph/cache/GsonGraphCache.kt b/common/src/main/java/com/ss/android/ugc/bytex/common/graph/cache/GsonGraphCache.kt index 959ee3e..8e9fc13 100644 --- a/common/src/main/java/com/ss/android/ugc/bytex/common/graph/cache/GsonGraphCache.kt +++ b/common/src/main/java/com/ss/android/ugc/bytex/common/graph/cache/GsonGraphCache.kt @@ -49,7 +49,13 @@ object GsonGraphCache : IGraphCache { } BufferedReader(FileReader(t)).use { reader -> GSON.fromJson>(reader, object : TypeToken>() { - }.type).parallelStream().forEach { graphBuilder.add(it) } + }.type).apply { + //经过抖音cache测试性能,结果多线程方案差不多，但大约结果是 + // Schedulers.COMPUTATION().submitAndAwait { } synchronized(caches) { caches.remove(t.absolutePath) ?: return false - }.parallelStream().forEach { - graphBuilder.add(it) + }.apply { + //经过抖音cache测试性能,结果多线程方案差不多，但大约结果是 + // Schedulers.COMPUTATION().submitAndAwait> upload.properties #./gradlew clean if [[ "$upload2JCenter" == "true" ]]; then + echo "upload2JCenter=$upload2JCenter" >> upload.properties ./gradlew bintrayUpload --stacktrace else diff --git a/refer-check-plugin/src/main/java/com/ss/android/ugc/bytex/refercheck/ReferCheckPlugin.java b/refer-check-plugin/src/main/java/com/ss/android/ugc/bytex/refercheck/ReferCheckPlugin.java index 8b5c338..5b17b10 100644 --- a/refer-check-plugin/src/main/java/com/ss/android/ugc/bytex/refercheck/ReferCheckPlugin.java +++ b/refer-check-plugin/src/main/java/com/ss/android/ugc/bytex/refercheck/ReferCheckPlugin.java @@ -2,10 +2,13 @@ import com.android.build.gradle.AppExtension; import com.ss.android.ugc.bytex.common.CommonPlugin; +import com.ss.android.ugc.bytex.common.flow.TransformFlow; +import com.ss.android.ugc.bytex.common.flow.main.MainTransformFlow; import com.ss.android.ugc.bytex.common.visitor.ClassVisitorChain; import com.ss.android.ugc.bytex.pluginconfig.anno.PluginConfig; import com.ss.android.ugc.bytex.refercheck.log.ErrorLogGenerator; import com.ss.android.ugc.bytex.refercheck.visitor.ReferCheckClassVisitor; +import com.ss.android.ugc.bytex.transformer.TransformContext; import com.ss.android.ugc.bytex.transformer.TransformEngine; import org.gradle.api.Project; @@ -47,4 +50,14 @@ public void afterTransform(@Nonnull TransformEngine engine) { } context.release(); } + + @Override + protected TransformFlow provideTransformFlow(@Nonnull MainTransformFlow mainFlow, @Nonnull TransformContext transformContext) { + return new MainTransformFlow(new TransformEngine(transformContext)) { + @Override + public int getPriority() { + return Integer.MIN_VALUE; + } + }.appendHandler(this); + } } diff --git a/wiki/ByteX-Developer-API-en.md b/wiki/ByteX-Developer-API-en.md index 8dcbc39..ce16060 100644 --- a/wiki/ByteX-Developer-API-en.md +++ b/wiki/ByteX-Developer-API-en.md @@ -142,7 +142,7 @@ PASSWORD_SNAPSHOT=xxx       After publishing the plugin locally and connecting it to the app project, append parameters at the end of the build-command before executing the build-command.For example: ``` -gradle clean :aweme:assembleRelease -Dorg.gradle.debug=true --no-daemon +./gradlew clean :example:assembleDouyinCnRelease -Dorg.gradle.debug=true --no-daemon ```       Then switch to the Configuration that you created just now, and click the debug button. diff --git a/wiki/ByteX-Developer-API-zh.md b/wiki/ByteX-Developer-API-zh.md index 3c84942..0c86020 100644 --- a/wiki/ByteX-Developer-API-zh.md +++ b/wiki/ByteX-Developer-API-zh.md @@ -140,7 +140,7 @@ PASSWORD_SNAPSHOT=xxx       把插件本地发布然后接入到app工程里之后,在命令行执行构建命令,末尾拼上这一串参数,比如： ``` -gradle clean :aweme:assembleRelease -Dorg.gradle.debug=true --no-daemon +./gradlew clean :example:assembleDouyinCnRelease -Dorg.gradle.debug=true --no-daemon ```       然后切换到刚刚创建的Configuration,点击debug按钮. From edf6ebaccb5fe71d7f8514401321fbd4a5c7bdb0 Mon Sep 17 00:00:00 2001 From: yangzhiqian Date: Mon, 15 Jun 2020 23:01:26 +0800 Subject: [PATCH 2/2] upload to jcenter:0.1.6 --- CHANGELOG.md | 6 ++++++ README.md | 2 +- README_zh.md | 2 +- build.gradle | 2 +- gradle/ext.gradle | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb15439..941ae20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Change Log +### Version 0.1.6 +- Upgrade AGP 3.5.3 +- Fix memory leak in HookProguard +- Run refer-check-plugin in single flow in order to check any issues producted by bytex'plugin +- Unified threadpool and run pipleline lifecycles in parallel + ### Version 0.1.5 - Fix bugs with getter-setter-inline plugins diff --git a/README.md b/README.md index adc3254..2b776f9 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Add those configuration code to your build.gradle, and apply your plugins on dem ```groovy buildscript { - ext.plugin_version="0.1.4" + ext.plugin_version="0.1.6" repositories { google() jcenter() diff --git a/README_zh.md b/README_zh.md index 2939fd0..4aef591 100644 --- a/README_zh.md +++ b/README_zh.md @@ -65,7 +65,7 @@ ByteX是一个基于gradle transform api和ASM的字节码插件平台（或许 ```groovy buildscript { - ext.plugin_version="0.1.4" + ext.plugin_version="0.1.6" repositories { google() jcenter() diff --git a/build.gradle b/build.gradle index fa02e5f..164d792 100644 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,7 @@ buildscript { classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" classpath "org.jetbrains.dokka:dokka-gradle-plugin:0.10.0" classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.3" - classpath "digital.wup:android-maven-publish:3.4.0" + classpath "digital.wup:android-maven-publish:3.6.3" } } diff --git a/gradle/ext.gradle b/gradle/ext.gradle index 930566b..7234ee5 100644 --- a/gradle/ext.gradle +++ b/gradle/ext.gradle @@ -32,7 +32,7 @@ ext { upload_password_snapshot = PASSWORD_SNAPSHOT upload_maven_url_snapshot = UPLOAD_MAVEN_URL_SNAPSHOT upload_group = 'com.bytedance.android.byteX' - upload_version = "0.1.5${(useSnapshotMaven.toBoolean() ? "-${doSh("git config user.name")}-SNAPSHOT" : "")}" + upload_version = "0.1.6${(useSnapshotMaven.toBoolean() ? "-${doSh("git config user.name")}-SNAPSHOT" : "")}" upload_dir = project.rootProject.file('gradle_plugins').path gradle_version = '3.5.3' asm_version = '6.2.1'