Classloader test detection (#82)

* Classloader test detection Signed-off-by: André Silva <[email protected]> * Fix class finder Signed-off-by: André Silva <[email protected]> * Fix conditional test ignore according to JVM Signed-off-by: André Silva <[email protected]>
ASSERT-KTH · Aug 13, 2021 · 2f2b59b · 2f2b59b
1 parent 14585dd
commit 2f2b59b
Show file tree

Hide file tree

Showing 36 changed files with 797 additions and 239 deletions.
diff --git a/pom.xml b/pom.xml
@@ -134,6 +134,13 @@
 			<version>3.11.2</version>
 			<scope>test</scope>
 		</dependency>
+
+		<!-- for test class detection -->
+		<dependency>
+			<groupId>org.apache.maven.surefire</groupId>
+			<artifactId>maven-surefire-common</artifactId>
+			<version>3.0.0-M5</version>
+		</dependency>
 	</dependencies>
 
 	<build>

diff --git a/src/main/java/fr/spoonlabs/flacoco/api/Suspiciousness.java b/src/main/java/fr/spoonlabs/flacoco/api/Suspiciousness.java
@@ -1,6 +1,6 @@
 package fr.spoonlabs.flacoco.api;
 
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 
 import java.util.List;
 

diff --git a/src/main/java/fr/spoonlabs/flacoco/cli/FlacocoMain.java b/src/main/java/fr/spoonlabs/flacoco/cli/FlacocoMain.java
@@ -109,6 +109,9 @@ public enum Format {
 		String customExporter;
 	}
 
+	@Option(names = {"--testDetectionStrategy"}, description = "Strategy for test detection stage. Defaults to CLASSLOADER. Valid values: ${COMPLETION-CANDIDATES}")
+	FlacocoConfig.TestDetectionStrategy testDetectionStrategy = FlacocoConfig.TestDetectionStrategy.CLASSLOADER;
+
 	@CommandLine.ArgGroup(exclusive = false, multiplicity = "0..1", heading = "Setting any of these options will result in test detection being bypassed.")
 	Tests tests = new Tests();
 
@@ -182,6 +185,7 @@ private void setupFlacocoConfig() {
 		config.setIncludeZeros(includeZeros);
 		config.setComplianceLevel(complianceLevel);
 
+		config.setTestDetectionStrategy(this.testDetectionStrategy);
 		config.setjUnit4Tests(this.tests.jUnit4Tests);
 		config.setjUnit5Tests(this.tests.jUnit5Tests);
 

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/config/FlacocoConfig.java b/src/main/java/fr/spoonlabs/flacoco/core/config/FlacocoConfig.java
@@ -19,6 +19,11 @@ public enum FaultLocalizationFamily {
 		SPECTRUM_BASED,
 	}
 
+	public enum TestDetectionStrategy {
+		TEST_RUNNER,
+		CLASSLOADER
+	}
+
 	private static FlacocoConfig instance;
 
 	private String workspace;
@@ -39,6 +44,7 @@ public enum FaultLocalizationFamily {
 	private boolean includeZeros;
 	private int complianceLevel;
 
+	private TestDetectionStrategy testDetectionStrategy;
 	private List<String> jUnit4Tests;
 	private List<String> jUnit5Tests;
 
@@ -76,6 +82,7 @@ private void initDefaults() {
 		this.includeZeros = false;
 		this.complianceLevel = 8;
 
+		this.testDetectionStrategy = TestDetectionStrategy.CLASSLOADER;
 		this.jUnit4Tests = new ArrayList<>();
 		this.jUnit5Tests = new ArrayList<>();
 
@@ -223,6 +230,14 @@ public void setTestRunnerJVMArgs(String testRunnerJVMArgs) {
 		this.testRunnerJVMArgs = testRunnerJVMArgs;
 	}
 
+	public TestDetectionStrategy getTestDetectionStrategy() {
+		return testDetectionStrategy;
+	}
+
+	public void setTestDetectionStrategy(TestDetectionStrategy testDetectionStrategy) {
+		this.testDetectionStrategy = testDetectionStrategy;
+	}
+
 	public List<String> getjUnit4Tests() {
 		return jUnit4Tests;
 	}
@@ -299,6 +314,7 @@ public String toString() {
 				", threshold=" + threshold +
 				", includeZero=" + includeZeros +
 				", complianceLevel=" + complianceLevel +
+				", testDetectionStrategy=" + testDetectionStrategy +
 				", jUnit4Tests='" + jUnit4Tests + '\'' +
 				", jUnit5Tests='" + jUnit5Tests + '\'' +
 				", family=" + family +

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageFromSingleTestUnit.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageFromSingleTestUnit.java
@@ -2,7 +2,7 @@
 
 import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod;
 import eu.stamp_project.testrunner.listener.impl.CoverageDetailed;
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 
 /**
  * Contains the results of the execution of a single test case (i.e., a method).

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageMatrix.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageMatrix.java
@@ -5,7 +5,7 @@
 import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod;
 import eu.stamp_project.testrunner.listener.impl.CoverageDetailed;
 import eu.stamp_project.testrunner.listener.impl.CoverageFromClass;
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 import org.apache.log4j.Logger;
 
 import java.util.*;

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageRunner.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/CoverageRunner.java
@@ -1,12 +1,9 @@
 package fr.spoonlabs.flacoco.core.coverage;
 
-import ch.scheitlin.alex.java.StackTrace;
-import ch.scheitlin.alex.java.StackTraceParser;
 import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod;
-import eu.stamp_project.testrunner.listener.impl.CoverageDetailed;
 import fr.spoonlabs.flacoco.core.config.FlacocoConfig;
 import fr.spoonlabs.flacoco.core.test.TestContext;
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 import org.apache.log4j.Logger;
 
 import java.util.List;

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/JUnit4Strategy.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/JUnit4Strategy.java
@@ -4,25 +4,37 @@
 import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod;
 import fr.spoonlabs.flacoco.core.config.FlacocoConfig;
 import fr.spoonlabs.flacoco.core.test.TestContext;
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 import org.apache.log4j.Logger;
 
 import java.util.concurrent.TimeoutException;
 
 public class JUnit4Strategy extends TestFrameworkStrategy {
 
-	private Logger logger = Logger.getLogger(JUnit4Strategy.class);
-	private FlacocoConfig config = FlacocoConfig.getInstance();
+	private static final Logger logger = Logger.getLogger(JUnit4Strategy.class);
+
+	private static JUnit4Strategy instance;
+
+	private JUnit4Strategy() {
+
+	}
+
+	public static JUnit4Strategy getInstance() {
+		if (instance == null) {
+			instance = new JUnit4Strategy();
+		}
+		return instance;
+	}
 
 	@Override
 	public CoveredTestResultPerTestMethod execute(TestContext testContext) throws TimeoutException {
-		this.logger.debug("Running " + testContext);
+		logger.debug("Running " + testContext);
 		this.setupTestRunnerEntryPoint();
 
 		return EntryPoint.runCoveredTestResultPerTestMethods(
 				this.computeClasspath(),
-				config.getBinJavaDir(),
-				config.getBinTestDir(),
+				FlacocoConfig.getInstance().getBinJavaDir(),
+				FlacocoConfig.getInstance().getBinTestDir(),
 				testContext.getTestMethods().stream().map(TestMethod::getFullyQualifiedClassName).distinct().toArray(String[]::new),
 				new String[0]
 		);

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/JUnit5Strategy.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/JUnit5Strategy.java
@@ -4,28 +4,40 @@
 import eu.stamp_project.testrunner.listener.CoveredTestResultPerTestMethod;
 import fr.spoonlabs.flacoco.core.config.FlacocoConfig;
 import fr.spoonlabs.flacoco.core.test.TestContext;
-import fr.spoonlabs.flacoco.core.test.TestMethod;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 import org.apache.log4j.Logger;
 
 import java.util.concurrent.TimeoutException;
 
 public class JUnit5Strategy extends TestFrameworkStrategy {
 
-	private Logger logger = Logger.getLogger(JUnit5Strategy.class);
-	private FlacocoConfig config = FlacocoConfig.getInstance();
+	private static final Logger logger = Logger.getLogger(JUnit5Strategy.class);
+
+	private static JUnit5Strategy instance;
+
+	private JUnit5Strategy() {
+
+	}
+
+	public static JUnit5Strategy getInstance() {
+		if (instance == null) {
+			instance = new JUnit5Strategy();
+		}
+		return instance;
+	}
 
 	@Override
 	public CoveredTestResultPerTestMethod execute(TestContext testContext) throws TimeoutException {
-		this.logger.debug("Running " + testContext);
+		logger.debug("Running " + testContext);
 		this.setupTestRunnerEntryPoint();
 
 		// test-runner needs a flag for JUnit5 tests
 		EntryPoint.jUnit5Mode = true;
 
 		return EntryPoint.runCoveredTestResultPerTestMethods(
 				this.computeClasspath(),
-				config.getBinJavaDir(),
-				config.getBinTestDir(),
+				FlacocoConfig.getInstance().getBinJavaDir(),
+				FlacocoConfig.getInstance().getBinTestDir(),
 				testContext.getTestMethods().stream().map(TestMethod::getFullyQualifiedClassName).distinct().toArray(String[]::new),
 				new String[0]
 		);

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/TestFrameworkStrategy.java b/src/main/java/fr/spoonlabs/flacoco/core/coverage/framework/TestFrameworkStrategy.java
@@ -13,23 +13,22 @@
 
 public abstract class TestFrameworkStrategy {
 
-	private Logger logger = Logger.getLogger(TestFrameworkStrategy.class);
-	private FlacocoConfig config = FlacocoConfig.getInstance();
-
+	private static final Logger logger = Logger.getLogger(TestFrameworkStrategy.class);
 
 	public abstract CoveredTestResultPerTestMethod execute(TestContext testContext) throws TimeoutException;
 
 	/**
 	 * Auxiliary method to setup test-runners default options
 	 */
 	protected void setupTestRunnerEntryPoint() {
+		FlacocoConfig config = FlacocoConfig.getInstance();
 		EntryPoint.coverageDetail = ParserOptions.CoverageTransformerDetail.DETAIL_COMPRESSED;
-		EntryPoint.workingDirectory = new File(this.config.getWorkspace());
-		EntryPoint.verbose = this.config.isTestRunnerVerbose();
-		EntryPoint.timeoutInMs = this.config.getTestRunnerTimeoutInMs();
-		EntryPoint.JVMArgs = this.config.getTestRunnerJVMArgs();
+		EntryPoint.workingDirectory = new File(config.getWorkspace());
+		EntryPoint.verbose = config.isTestRunnerVerbose();
+		EntryPoint.timeoutInMs = config.getTestRunnerTimeoutInMs();
+		EntryPoint.JVMArgs = config.getTestRunnerJVMArgs();
 		EntryPoint.jUnit5Mode = false;
-		if (this.config.isCoverTests()) {
+		if (config.isCoverTests()) {
 			throw new UnsupportedOperationException();
 		}
 	}
@@ -40,8 +39,9 @@ protected void setupTestRunnerEntryPoint() {
 	 * @return Classpath for test-runner execution
 	 */
 	protected String computeClasspath() {
-		String classpath = this.config.getClasspath();
-		String mavenHome = this.config.getMavenHome();
+		FlacocoConfig config = FlacocoConfig.getInstance();
+		String classpath = config.getClasspath();
+		String mavenHome = config.getMavenHome();
 		String junitClasspath;
 		String jacocoClassPath;
 
@@ -59,11 +59,11 @@ protected String computeClasspath() {
 		jacocoClassPath = mavenHome + "org/jacoco/org.jacoco.core/0.8.7/org.jacoco.core-0.8.7.jar";
 
 		// Add JUnit dependencies
-		if (this.config.getCustomJUnitClasspath() != null)
-			junitClasspath = this.config.getCustomJUnitClasspath();
+		if (config.getCustomJUnitClasspath() != null)
+			junitClasspath = config.getCustomJUnitClasspath();
 		// Add jacoco dependencies
-		if (this.config.getCustomJacocoClasspath() != null)
-			jacocoClassPath = this.config.getCustomJacocoClasspath();
+		if (config.getCustomJacocoClasspath() != null)
+			jacocoClassPath = config.getCustomJacocoClasspath();
 
 		return junitClasspath + File.pathSeparatorChar
 				+ jacocoClassPath + File.pathSeparatorChar

diff --git a/src/main/java/fr/spoonlabs/flacoco/core/test/SpoonTestMethod.java b/src/main/java/fr/spoonlabs/flacoco/core/test/SpoonTestMethod.java
diff --git a/src/main/java/fr/spoonlabs/flacoco/core/test/StringTestMethod.java b/src/main/java/fr/spoonlabs/flacoco/core/test/StringTestMethod.java
diff --git a/src/main/java/fr/spoonlabs/flacoco/core/test/TestContext.java b/src/main/java/fr/spoonlabs/flacoco/core/test/TestContext.java
@@ -1,6 +1,7 @@
 package fr.spoonlabs.flacoco.core.test;
 
 import fr.spoonlabs.flacoco.core.coverage.framework.TestFrameworkStrategy;
+import fr.spoonlabs.flacoco.core.test.method.TestMethod;
 
 import java.util.ArrayList;
 import java.util.Collection;