Merge pull request #41 from bytedance/yangzhiqian/master

Upgrade AGP(3.5.3)、Fix memory leak、Run refer-check in single、Unified threadpool

CHANGELOG.md

@@ -1,4 +1,10 @@
 # Change Log
+### Version 0.1.6
+- Upgrade AGP 3.5.3
+- Fix memory leak in HookProguard
+- Run refer-check-plugin in single flow in order to check any issues producted by bytex'plugin
+- Unified threadpool and run pipleline lifecycles in parallel
+
 ### Version 0.1.5
 - Fix bugs with getter-setter-inline plugins
 

GradleToolKit/GradleEnvApi/build.gradle

@@ -3,7 +3,7 @@ group "$upload_group"
 version "$upload_version"
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
-    compile gradleApi()
+    compileOnly gradleApi()
     compileOnly "com.android.tools.build:gradle:$gradle_version"
 }
 sourceCompatibility = "1.8"

GradleToolKit/build.gradle

@@ -6,15 +6,15 @@ group "$upload_group"
 version "$upload_version"
 dependencies {
     kapt 'com.google.auto.service:auto-service:1.0-rc4'
-    compile gradleApi()
+    compileOnly gradleApi()
     compileOnly "com.android.tools.build:gradle:$gradle_version"
     compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     compile project(':GradleEnvApi')
 
-    compile ('com.google.auto.service:auto-service:1.0-rc4') {
+    compile('com.google.auto.service:auto-service:1.0-rc4') {
         exclude module: 'guava'
     }
-    implementation "com.didiglobal.booster:booster-android-gradle-api:1.2.0"
+    implementation "com.didiglobal.booster:booster-android-gradle-api:2.0.0"
 }
 sourceCompatibility = "1.8"
 targetCompatibility = "1.8"

GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeResources.kt

@@ -5,29 +5,48 @@ package com.ss.android.ugc.bytex.gradletoolkit
  */
 
 import com.android.build.gradle.tasks.MergeResources
+import com.android.ide.common.resources.ResourceSet
 import java.io.File
 
 fun MergeResources.resourceSetList(): List<File> {
+    val resourceSets = try {
+        resourceSetList1()
+    } catch (e: Exception) {
+        resourceSetList2()
+    }
+    return resourceSets.flatMap { it.sourceFiles }.toSet().toList()
+}
+
+
+fun MergeResources.resourceSetList1(): Iterable<ResourceSet> {
 
     val computeResourceSetListMethod = MergeResources::class.java.declaredMethods
-            .firstOrNull { it.name == "computeResourceSetList" && it.parameterCount == 0 }
-            ?: return emptyList()
+            .find { it.name == "computeResourceSetList" && it.parameterCount == 0 }!!
 
     val oldIsAccessible = computeResourceSetListMethod.isAccessible
     try {
         computeResourceSetListMethod.isAccessible = true
+        return computeResourceSetListMethod.invoke(this) as Iterable<ResourceSet>
+    } finally {
+        computeResourceSetListMethod.isAccessible = oldIsAccessible
+    }
+}
 
-        val resourceSets = computeResourceSetListMethod.invoke(this) as? Iterable<*>
-                ?: return emptyList()
+fun MergeResources.resourceSetList2(): Iterable<ResourceSet> {
+    val getConfiguredResourceSets = MergeResources::class.java.declaredMethods
+            .find { it.name == "getConfiguredResourceSets" && it.parameterCount == 1 }!!
 
-        return resourceSets.mapNotNull { resourceSet ->
-            val getSourceFiles = resourceSet?.javaClass?.methods?.find { it.name == "getSourceFiles" && it.parameterCount == 0 }
-            val files = getSourceFiles?.invoke(resourceSet)
-            @Suppress("UNCHECKED_CAST")
-            files as? Iterable<File>
-        }.flatten()
+    val getPreprocessor = MergeResources::class.java.declaredMethods
+            .find { it.name == "getPreprocessor" && it.parameterCount == 0 }!!
 
+    val getConfiguredResourceSetsAccess = getConfiguredResourceSets.isAccessible
+    val getPreprocessorAccess = getPreprocessor.isAccessible
+    try {
+        getConfiguredResourceSets.isAccessible = true
+        getPreprocessor.isAccessible = true
+        return getConfiguredResourceSets.invoke(this, getPreprocessor.invoke(this)) as Iterable<ResourceSet>
     } finally {
-        computeResourceSetListMethod.isAccessible = oldIsAccessible
+        getConfiguredResourceSets.isAccessible = getConfiguredResourceSetsAccess
+        getPreprocessor.isAccessible = getPreprocessorAccess
     }
 }

GradleToolKit/src/main/kotlin/com/ss/android/ugc/bytex/gradletoolkit/MergeSourceSetFolders.kt

@@ -1,6 +1,7 @@
 package com.ss.android.ugc.bytex.gradletoolkit
 
 import com.android.build.gradle.tasks.MergeSourceSetFolders
+import com.android.ide.common.resources.AssetSet
 import java.io.File
 
 /**
@@ -9,27 +10,37 @@ import java.io.File
 
 
 fun MergeSourceSetFolders.assetSetList(): List<File> {
+    val assetSets = try {
+        assetSetList1()
+    } catch (e: Exception) {
+        assetSetList2()
+    }
+    return assetSets.flatMap { it.sourceFiles }.toSet().toList()
+}
+
 
+fun MergeSourceSetFolders.assetSetList1(): Iterable<AssetSet> {
     val computeAssetSetListMethod = MergeSourceSetFolders::class.java.declaredMethods
-            .firstOrNull { it.name == "computeAssetSetList" && it.parameterCount == 0 }
-            ?: return emptyList()
+            .find { it.name == "computeAssetSetList" && it.parameterCount == 0 }!!
 
     val oldIsAccessible = computeAssetSetListMethod.isAccessible
     try {
         computeAssetSetListMethod.isAccessible = true
+        return computeAssetSetListMethod.invoke(this) as Iterable<AssetSet>
+    } finally {
+        computeAssetSetListMethod.isAccessible = oldIsAccessible
+    }
+}
 
-        val assetSets = computeAssetSetListMethod.invoke(this) as? Iterable<*>
-                ?: return emptyList()
-
-        return assetSets.mapNotNull { assetSet ->
-            val getSourceFiles = assetSet?.javaClass?.methods?.find { it.name == "getSourceFiles" && it.parameterCount == 0 }
-            val files = getSourceFiles?.invoke(assetSet)
-            @Suppress("UNCHECKED_CAST")
-            files as? Iterable<File>
-        }.flatten()
+fun MergeSourceSetFolders.assetSetList2(): Iterable<AssetSet> {
+    val computeAssetSetListMethod = MergeSourceSetFolders::class.java.declaredMethods
+            .find { it.name == "computeAssetSetList\$gradle" && it.parameterCount == 0 }!!
 
+    val oldIsAccessible = computeAssetSetListMethod.isAccessible
+    try {
+        computeAssetSetListMethod.isAccessible = true
+        return computeAssetSetListMethod.invoke(this) as Iterable<AssetSet>
     } finally {
         computeAssetSetListMethod.isAccessible = oldIsAccessible
     }
-
-}
+}

HookProguard/src/main/java/com/ss/android/ugc/bytex/hookproguard/KeepClassSpecificationHolder.java

@@ -15,7 +15,6 @@ class KeepClassSpecificationHolder {
     private final KeepClassSpecification instance;
     private StringMatcher classNameMatcher;
     private List<MemberSpecificationHolder> methodSpecifications;
-    private Node extendsClassNode;
 
     KeepClassSpecificationHolder(KeepClassSpecification instance, StringParser parser) {
         this.instance = instance;
@@ -50,9 +49,6 @@ void parserMethodSpecifications(StringParser parser) {
     }
 
     Node computeExtendsClassNode(Graph graph) {
-        if (extendsClassNode == null) {
-            extendsClassNode = graph.get(instance.extendsClassName);
-        }
-        return extendsClassNode;
+        return graph.get(instance.extendsClassName);
     }
 }

PluginConfig/PluginConfigProcessor/build.gradle

@@ -3,6 +3,7 @@ apply plugin: 'java'
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
     compile('com.google.auto.service:auto-service:1.0-rc4')
+    annotationProcessor 'com.google.auto.service:auto-service:1.0-rc4'
     compile project(':PluginConfigAnnotation')
 }
 sourceCompatibility = "1.8"

README.md

@@ -74,7 +74,7 @@ Add those configuration code to your build.gradle, and apply your plugins on dem
 
 ```groovy
 buildscript {
-    ext.plugin_version="0.1.4"
+    ext.plugin_version="0.1.6"
     repositories {
         google()
         jcenter()

README_zh.md

@@ -65,7 +65,7 @@ ByteX是一个基于gradle transform api和ASM的字节码插件平台（或许
 
 ```groovy
 buildscript {
-    ext.plugin_version="0.1.4"
+    ext.plugin_version="0.1.6"
     repositories {
         google()
         jcenter()

TransformEngine/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'java-library'
 apply plugin: 'kotlin'
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
-    compile gradleApi()
+    compileOnly gradleApi()
     compileOnly "com.android.tools.build:gradle:$gradle_version"
     compile "com.google.guava:guava:$guava_version"
     testCompile 'com.squareup.okio:okio:1.14.0'

TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformContext.java

@@ -120,7 +120,8 @@ public boolean shouldSaveCache() {
     }
 
     /**
-     * 请求非增量运行，必须在traverse时机之前调用
+     * 请求非增量运行，必须在traverse时机之前调用<br/>
+     * beforeTraverse及之前生命周期可调用，否则报RuntimeException<br/>
      */
     public void requestNotIncremental() {
         if (running.get()) {
@@ -130,6 +131,23 @@ public void requestNotIncremental() {
         this.transformInputs.requestNotIncremental();
     }
 
+    /**
+     * 请求某个文件进行非增量<br/>
+     * beforeTraverse及之前生命周期可调用，否则报RuntimeException<br/>
+     *
+     * @param relativePath 文件的相对路径,比如 com/bytedance/Demo.class
+     * @return 成功修改对应输入的状态，如果当前已经是非增量
+     */
+    public boolean requestNotIncremental(String relativePath) {
+        if (running.get()) {
+            throw new RuntimeException("You Should request for not incremental before traversing.");
+        }
+        if (!this.isIncremental()) {
+            return false;
+        }
+        return this.transformInputs.requestNotIncremental(relativePath);
+    }
+
     public boolean isReleaseBuild() {
         return invocation.getContext().getVariantName().toLowerCase().contains("release");
     }

TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformEngine.java

@@ -28,11 +28,12 @@ public class TransformEngine {
     public TransformEngine(TransformContext context) {
         this.context = context;
     }
-    public void beginRun(){
+
+    public void beginRun() {
         context.markRunningState(false);
     }
 
-    public void running(){
+    public void running() {
         context.markRunningState(true);
     }
 
@@ -55,6 +56,18 @@ public void skip() throws IOException {
         worker.await();
     }
 
+    public void transformOutput() throws IOException {
+        Worker worker = Schedulers.IO();
+        context.allFiles()
+                .filter(fileCache -> !fileCache.isHasWritten())
+                .map(f -> (Callable<Void>) () -> {
+                    f.transformOutput();
+                    return null;
+                })
+                .forEach(worker::submit);
+        worker.await();
+    }
+
     public void traverseOnly(FileProcessor... processors) {
         Schedulers.FORKJOINPOOL().invoke(new PerformTraverseTask(context.allFiles(), getProcessorList(processors)));
     }
@@ -70,9 +83,10 @@ private static List<FileProcessor> getProcessorList(FileProcessor[] processors)
         return realProcessorList;
     }
 
-    public void endRun(){
+    public void endRun() {
 
     }
+
     public TransformContext getContext() {
         return context;
     }

TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/TransformInputs.kt

@@ -116,9 +116,22 @@ class TransformInputs internal constructor(private val context: TransformContext
     }
 
     protected fun requestNotIncremental() {
-        transformInputs.flatMap { it.value }.parallelStream().forEach {
-            it.status = Status.ADDED
+        transformInputs.flatMap { it.value }.forEach {
+            if (it.status == Status.NOTCHANGED) {
+                it.status = Status.CHANGED
+            }
+        }
+    }
+
+    protected fun requestNotIncremental(relativePath: String): Boolean {
+        var r = false
+        transformInputs.flatMap { it.value }.forEach {
+            if (it.relativePath == relativePath && it.status == Status.NOTCHANGED) {
+                it.status = Status.CHANGED
+                r = true
+            }
         }
+        return r
     }
 
     fun addFile(affinity: String, file: FileData) {

TransformEngine/src/main/java/com/ss/android/ugc/bytex/transformer/cache/DirCache.java

@@ -50,7 +50,10 @@ public File getFile() {
     }
 
     @Override
-    public void transformOutput(Consumer<FileData> visitor) throws IOException {
+    public final synchronized void transformOutput(Consumer<FileData> visitor) throws IOException {
+        if (hasWritten) {
+            throw new RuntimeException("rewrite");
+        }
         Map<String, TransformOutputs.Entry> entryMap = new HashMap<>();
         String relativeToProject = context.getTransformOutputs().relativeToProject(outputFile);
         TransformOutputs.Entry entry = context.getTransformOutputs().getLastTransformOutputs().get(relativeToProject);
@@ -59,7 +62,7 @@ public void transformOutput(Consumer<FileData> visitor) throws IOException {
             entry.traverseAll(e -> entryMap.put(e.getPath(), e));
         }
         List<TransformOutputs.Entry> entries = Collections.synchronizedList(new LinkedList<>());
-        parallelForEach(true, item -> {
+        parallelForEach(false, item -> {
             if (visitor != null) visitor.accept(item);
             entries.add(
                     transformOutput(
@@ -76,6 +79,7 @@ public void transformOutput(Consumer<FileData> visitor) throws IOException {
                         0L,
                         Collections.unmodifiableList(entries))
         );
+        hasWritten = true;
     }
 
     private TransformOutputs.Entry transformOutput(String input, String parent, FileData fileData, Map<String, TransformOutputs.Entry> entryMap) throws IOException {
@@ -183,7 +187,11 @@ protected List<FileData> resolve(ObservableEmitter<FileData> emitter) throws IOE
     }
 
     @Override
-    public void skip() throws IOException {
+    public synchronized void skip() throws IOException {
+        if (hasWritten) {
+            throw new RuntimeException("rewrite");
+        }
         FileUtils.copyDirectory(getFile(), outputFile);
+        hasWritten = true;
     }
 }