diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36d6165f..c69aa5b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: run: sbt "project tinkergraph" test - name: Run OverflowDB tests run: sbt "project overflowdb" test - - name: Run Neo4j tests - run: sbt "project neo4j" test +# - name: Run Neo4j tests +# run: sbt "project neo4j" test - name: Run Neo4j Embedded tests run: sbt "project neo4jEmbed" test # - name: Run TigerGraph tests diff --git a/.gitignore b/.gitignore index 206e383b..00c70673 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,12 @@ /build/ /bin/ *.bin +*.cpg *.gzip graph.xml gsql_client.* +*.txt +*.csv # Ignore Gradle GUI config gradle-app.setting @@ -32,3 +35,4 @@ gradle-app.setting **/.bsp/ target /project/target/ +/neo4j-db diff --git a/README.md b/README.md index 370d53b3..a07fa86b 100644 --- a/README.md +++ b/README.md @@ -28,17 +28,51 @@ research to begin on Joern where I also spend time providing help and support. ## Quickstart One can run Plume from the `plume` binary which will use `OverflowDB` as the graph database backend if no config is -found. If one would like to configure another backend then the example `driver.yaml` can be adjusted to include the use -of another database by uncommenting and editing the respective fields given by the skeleton. E.g. using Neo4j: - -```yaml -database: Neo4j -params: - hostname: localhost - port: 7687 - username: neo4j - password: neo4j - txMax: 25 +found. If one would like to configure another backend then use the CLI arguments: + +```bash +Usage: plume [tinkergraph|overflowdb|neo4j|neo4j-embedded|tigergraph|neptune] [options] input-dir + +An AST creator for comparing graph databases as static analysis backends. + -h, --help + input-dir The target application to parse. +Command: tinkergraph [options] + + --import-path The TinkerGraph to import. + --export-path The TinkerGraph export path to serialize the result to. +Command: overflowdb [options] + + --storage-location + --heap-percentage-threshold + --enable-serialization-stats +Command: neo4j [options] + + --hostname + --port + --username + --password + --tx-max +Command: neo4j-embedded [options] + + --databaseName + --databaseDir + --tx-max +Command: tigergraph [options] + + --hostname + --restpp-port + --gsql-port + --username + --password + --timeout + --tx-max + --scheme +Command: neptune [options] + + --hostname + --port + --key-cert-chain-file + --tx-max ``` For more documentation and basic guides, check out the [project homepage](https://plume-oss.github.io/plume-docs/) or @@ -100,6 +134,15 @@ sbt stage This will build `target/scala-2.13/plume_2.13-X.X.X.jar` which can be imported into your local project. +## Benchmarks + +Plume specifies a `benchmark` binary which orchestrates running JMH benchmarks for AST creation with various graph +database backends. While the binary explains the available functions, the execution should be run within `sbt`, e.g. + +```sbt +Jmh/runMain com.github.plume.oss.Benchmark overflowdb testprogram -o output -r results --storage-location test.cpg +``` + ## Logging Plume uses [SLF4J](http://www.slf4j.org/) as the logging fascade. diff --git a/astcreator/build.sbt b/astcreator/build.sbt new file mode 100644 index 00000000..f5fb23f9 --- /dev/null +++ b/astcreator/build.sbt @@ -0,0 +1,11 @@ +name := "astcreator" + +dependsOn(Projects.base, Projects.commons, Projects.gremlin, Projects.overflowdb) + +libraryDependencies ++= Seq( + "io.joern" %% "semanticcpg" % Versions.joern, + "io.joern" %% "x2cpg" % Versions.joern, + "io.joern" %% "jimple2cpg" % Versions.joern, + "io.joern" %% "jimple2cpg" % Versions.joern % Test classifier "tests", + "io.joern" %% "x2cpg" % Versions.joern % Test classifier "tests", +) diff --git a/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala b/astcreator/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala similarity index 100% rename from src/main/scala/com/github/plume/oss/JimpleAst2Database.scala rename to astcreator/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala diff --git a/src/main/scala/com/github/plume/oss/passes/IncrementalKeyPool.scala b/astcreator/src/main/scala/com/github/plume/oss/passes/IncrementalKeyPool.scala similarity index 100% rename from src/main/scala/com/github/plume/oss/passes/IncrementalKeyPool.scala rename to astcreator/src/main/scala/com/github/plume/oss/passes/IncrementalKeyPool.scala diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala b/astcreator/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala similarity index 100% rename from src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala rename to astcreator/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala diff --git a/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala b/astcreator/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala similarity index 100% rename from src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala rename to astcreator/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala diff --git a/benchmark b/benchmark new file mode 120000 index 00000000..8381fa18 --- /dev/null +++ b/benchmark @@ -0,0 +1 @@ +target/universal/stage/bin/benchmark \ No newline at end of file diff --git a/build.sbt b/build.sbt index a86d88b1..338822f9 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,14 @@ inThisBuild( organization := "com.github.plume-oss", version := "2.0.0", scalaVersion := "3.4.2", - resolvers ++= Seq(Resolver.mavenLocal, Resolver.mavenCentral, Resolver.JCenterRepository) + resolvers ++= Seq( + Resolver.mavenLocal, + Resolver.mavenCentral, + Resolver.JCenterRepository, + "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public", + "Atlassian" at "https://packages.atlassian.com/mvn/maven-atlassian-external", + "Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases/" + ) ) ) @@ -20,28 +27,30 @@ lazy val neo4j = Projects.neo4j lazy val neo4jEmbed = Projects.neo4jEmbed lazy val tigergraph = Projects.tigergraph lazy val overflowDb = Projects.overflowdb +// AST creation +lazy val astcreator = Projects.astcreator lazy val root = (project in file(".")) - .dependsOn(commons, base, gremlin, tinkergraph, neptune, neo4j, neo4jEmbed, tigergraph, overflowDb) - .aggregate(commons, base, gremlin, tinkergraph, neptune, neo4j, neo4jEmbed, tigergraph, overflowDb) + .dependsOn(astcreator, commons, base, gremlin, tinkergraph, neptune, neo4j, neo4jEmbed, tigergraph, overflowDb) + .aggregate(astcreator, commons, base, gremlin, tinkergraph, neptune, neo4j, neo4jEmbed, tigergraph, overflowDb) trapExit := false Test / fork := true Test / parallelExecution := false libraryDependencies ++= Seq( - "io.joern" %% "semanticcpg" % Versions.joern, - "io.joern" %% "x2cpg" % Versions.joern, - "io.joern" %% "jimple2cpg" % Versions.joern, - "io.joern" %% "jimple2cpg" % Versions.joern % Test classifier "tests", - "io.joern" %% "x2cpg" % Versions.joern % Test classifier "tests", - "org.slf4j" % "slf4j-api" % Versions.slf4j, - "org.apache.logging.log4j" % "log4j-core" % Versions.log4j % Test, - "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j % Test, - "org.scalatest" %% "scalatest" % Versions.scalatest % Test + "org.openjdk.jmh" % "jmh-generator-annprocess" % Versions.jmh, + "org.openjdk.jmh" % "jmh-core" % Versions.jmh, + "org.openjdk.jmh" % "jmh-generator-bytecode" % Versions.jmh, + "org.openjdk.jmh" % "jmh-generator-reflection" % Versions.jmh, + "org.openjdk.jmh" % "jmh-generator-asm" % Versions.jmh, + "org.slf4j" % "slf4j-api" % Versions.slf4j, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j % Test, + "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j % Test, + "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) -enablePlugins(JavaAppPackaging) +enablePlugins(JavaAppPackaging, JmhPlugin) scmInfo := Some(ScmInfo(url("https://github.com/plume-oss/plume"), "git@github.com:plume-oss/plume.git")) diff --git a/commons/src/main/scala/com/github/plume/oss/PlumeStatistics.scala b/commons/src/main/scala/com/github/plume/oss/PlumeStatistics.scala deleted file mode 100644 index 3bacafaf..00000000 --- a/commons/src/main/scala/com/github/plume/oss/PlumeStatistics.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.github.plume.oss - -import scala.collection.mutable - -/** Holds statistics regarding the performance of certain stages of the extraction process. - */ -object PlumeStatistics extends Enumeration { - - /** Represents a measureable feature that can be timed. - */ - type PlumeStatistic = Value - - val TIME_OPEN_DRIVER, TIME_CLOSE_DRIVER, TIME_EXTRACTION, TIME_REACHABLE_BY_QUERYING, TIME_REMOVING_OUTDATED_GRAPH, - TIME_REMOVING_OUTDATED_CACHE, TIME_RETRIEVING_CACHE, TIME_STORING_CACHE, PROGRAM_CLASSES, PROGRAM_METHODS = Value - - private val statistics: mutable.Map[PlumeStatistic, Long] = - PlumeStatistics.values.map((_, 0L)).to(collection.mutable.Map) - - /** Measures the time for a given code block and saves it under the [[PlumeStatistic]] heading. - * @param statistic - * the statistic to save the elapsed time under. - * @param block - * the code block to measure. - * @tparam R - * the return value of the code block. - * @return - * the result of the code block. - */ - def time[R](statistic: PlumeStatistic, block: => R): R = { - val t0 = System.nanoTime() - val result = block - val t1 = System.nanoTime() - statistics.put(statistic, t1 - t0) - result - } - - /** Sets the value of a statistic. - * @param statistic - * The statistic to set. - * @param newValue - * The new value to associate with the statistic. - */ - def setMetric(statistic: PlumeStatistic, newValue: Long): Unit = - statistics.put(statistic, newValue) - - /** The results of the measured time per process. - * @return - * a map of each [[PlumeStatistic]] and the time per process in nanoseconds. - */ - def results(): Map[PlumeStatistic, Long] = statistics.toMap - - /** Sets all the clocks back to 0. - */ - def reset(): Unit = { - statistics.clear() - PlumeStatistics.values.map((_, 0L)).foreach { case (v, t) => statistics.put(v, t) } - } - -} diff --git a/driver.yaml b/driver.yaml deleted file mode 100644 index 44ea4f8a..00000000 --- a/driver.yaml +++ /dev/null @@ -1,37 +0,0 @@ -#database: OverflowDB -#params: -# storageLocation: cpg.odb -# heapPercentageThreshold: 80 -# serializationStatsEnabled: false -# maxCallDepth: 2 - -database: TinkerGraph -params: - # used for incremental updates if present -# importPath: graph.xml - exportPath: graph.xml - -#database: Neo4j -#params: -# hostname: localhost -# port: 7687 -# username: neo4j -# password: neo4j -# txMax: 25 - -#database: TigerGraph -#params: -# hostname: localhost -# restPpPort: 9000 -# gsqlPort: 14240 -# username: tigergraph -# password: tigergraph -# timeout: 3000 # 30 seconds -# txMax: 25 - -#database: Neptune -#params: -# hostname: instance_id.region_name.neptune.amazonaws.com -# port: 8182 -# keyCertChainFile: path/to/cert.pem -# txMax: 50 \ No newline at end of file diff --git a/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala b/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala index 9aa0c0eb..c9aaf917 100644 --- a/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala +++ b/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala @@ -13,6 +13,7 @@ import overflowdb.{BatchedUpdate, DetachedNodeGeneric} import scala.jdk.CollectionConverters.IteratorHasAsScala import scala.language.postfixOps +import scala.util.Try class PlumeDriverFixture(val driver: IDriver) extends AnyWordSpec @@ -68,7 +69,12 @@ class PlumeDriverFixture(val driver: IDriver) val changes = diffGraph1.iterator.asScala.toList val srcNode = changes .collectFirst { - case c: DetachedNodeGeneric if c.getRefOrId == m.getOrElse("id", -1L).toString.toLong => + case c: DetachedNodeGeneric + if c.getRefOrId == m.getOrElse("id", -1L).toString.toLong || Try( + c.getRefOrId + .asInstanceOf[StoredNode] + .id() + ).map(_ == m.getOrElse("id", -1L).toString.toLong).getOrElse(false) => c } match { case Some(src) => src @@ -76,7 +82,14 @@ class PlumeDriverFixture(val driver: IDriver) } val dstNode = changes .collectFirst { - case c: NewBlock if c.getRefOrId().asInstanceOf[Long] == b.getOrElse("id", -1L).toString.toLong => c + case c: NewBlock + if Try(c.getRefOrId().asInstanceOf[Long]) + .map(_ == b.getOrElse("id", -1L).toString.toLong) + .getOrElse(false) || + Try(c.getRefOrId().asInstanceOf[StoredNode].id()) + .map(_ == b.getOrElse("id", -1L).toString.toLong) + .getOrElse(false) => + c } match { case Some(dst) => dst case None => fail("Unable to extract block node") diff --git a/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala b/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala index 7681995d..3b7591b1 100644 --- a/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala +++ b/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala @@ -1,10 +1,8 @@ package com.github.plume.oss.drivers -import com.github.plume.oss.PlumeStatistics import com.github.plume.oss.util.BatchedUpdateUtil -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} +import io.shiftleft.codepropertygraph.generated.PropertyNames import org.apache.commons.configuration.BaseConfiguration -import org.apache.tinkerpop.gremlin.process.traversal.P.within import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.{coalesce, constant, values} import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.{GraphTraversal, GraphTraversalSource, __} import org.apache.tinkerpop.gremlin.structure.{Edge, Graph, T, Vertex} @@ -14,7 +12,7 @@ import overflowdb.BatchedUpdate.{Change, DiffOrBuilder} import overflowdb.{BatchedUpdate, DetachedNodeData} import java.util.concurrent.atomic.AtomicBoolean -import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala, MapHasAsScala} +import scala.jdk.CollectionConverters.{IteratorHasAsScala, MapHasAsScala} import scala.util.{Failure, Success, Try} /** The driver used by databases implementing Gremlin. @@ -25,15 +23,14 @@ abstract class GremlinDriver(txMax: Int = 50) extends IDriver { protected val config: BaseConfiguration = new BaseConfiguration() config.setProperty("gremlin.graph", "org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph") config.setProperty("gremlin.tinkergraph.vertexIdManager", "LONG") - protected val graph: Graph = - PlumeStatistics.time(PlumeStatistics.TIME_OPEN_DRIVER, { TinkerGraph.open(config) }) + protected val graph: Graph = TinkerGraph.open(config) protected var traversalSource: Option[GraphTraversalSource] = None private val connected = new AtomicBoolean(true) override def isConnected: Boolean = connected.get() override def close(): Unit = - Try(PlumeStatistics.time(PlumeStatistics.TIME_CLOSE_DRIVER, { graph.close() })) match { + Try(graph.close()) match { case Success(_) => connected.set(false) case Failure(e) => logger.warn("Exception thrown while attempting to close graph.", e) diff --git a/drivers/neo4j-embedded/src/main/scala/com/github/plume/oss/drivers/Neo4jEmbeddedDriver.scala b/drivers/neo4j-embedded/src/main/scala/com/github/plume/oss/drivers/Neo4jEmbeddedDriver.scala index 5d2ccb8b..6dab0dc0 100644 --- a/drivers/neo4j-embedded/src/main/scala/com/github/plume/oss/drivers/Neo4jEmbeddedDriver.scala +++ b/drivers/neo4j-embedded/src/main/scala/com/github/plume/oss/drivers/Neo4jEmbeddedDriver.scala @@ -1,7 +1,6 @@ package com.github.plume.oss.drivers import better.files.File -import com.github.plume.oss.PlumeStatistics import com.github.plume.oss.drivers.Neo4jEmbeddedDriver.* import com.github.plume.oss.util.BatchedUpdateUtil.* import io.shiftleft.codepropertygraph.generated.nodes.StoredNode @@ -32,8 +31,7 @@ final class Neo4jEmbeddedDriver( private val connected = new AtomicBoolean(true) private var managementService = new DatabaseManagementServiceBuilder(databaseDir.path).build() registerShutdownHook(managementService) - private var graphDb = - PlumeStatistics.time(PlumeStatistics.TIME_OPEN_DRIVER, { managementService.database(databaseName) }) + private var graphDb = managementService.database(databaseName) private def registerShutdownHook(managementService: DatabaseManagementService): Unit = { Runtime.getRuntime.addShutdownHook(new Thread() { @@ -45,7 +43,7 @@ final class Neo4jEmbeddedDriver( private def connect(): Unit = { managementService = new DatabaseManagementServiceBuilder(databaseDir.path).build() - graphDb = PlumeStatistics.time(PlumeStatistics.TIME_OPEN_DRIVER, { managementService.database(databaseName) }) + graphDb = managementService.database(databaseName) connected.set(true) } @@ -57,14 +55,11 @@ final class Neo4jEmbeddedDriver( connect() } - override def close(): Unit = PlumeStatistics.time( - PlumeStatistics.TIME_CLOSE_DRIVER, { - Try(managementService.shutdown()) match { - case Failure(e) => logger.warn("Exception thrown while attempting to close graph.", e) - case Success(_) => connected.set(false) - } + override def close(): Unit = + Try(managementService.shutdown()) match { + case Failure(e) => logger.warn("Exception thrown while attempting to close graph.", e) + case Success(_) => connected.set(false) } - ) override def exists(nodeId: Long): Boolean = Using.resource(graphDb.beginTx) { tx => diff --git a/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala b/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala index 7216329a..f9fa7957 100644 --- a/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala +++ b/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala @@ -1,18 +1,16 @@ package com.github.plume.oss.drivers -import com.github.plume.oss.PlumeStatistics import com.github.plume.oss.drivers.Neo4jDriver.* import com.github.plume.oss.util.BatchedUpdateUtil.* -import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} -import org.neo4j.driver.{AuthTokens, GraphDatabase, Transaction, Value} +import io.shiftleft.codepropertygraph.generated.nodes.StoredNode +import org.neo4j.driver.types.TypeSystem +import org.neo4j.driver.{AuthTokens, GraphDatabase, Transaction} import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate.{AppliedDiff, CreateEdge, DiffOrBuilder, SetNodeProperty} +import overflowdb.BatchedUpdate.{CreateEdge, DiffOrBuilder, SetNodeProperty} import overflowdb.{BatchedUpdate, DetachedNodeData} import java.util import java.util.concurrent.atomic.AtomicBoolean -import scala.collection.mutable import scala.jdk.CollectionConverters import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala} import scala.util.{Failure, Success, Try, Using} @@ -29,19 +27,15 @@ final class Neo4jDriver( ) extends IDriver with ISchemaSafeDriver { - private val logger = LoggerFactory.getLogger(classOf[Neo4jDriver]) - private val connected = new AtomicBoolean(true) - private val driver = - PlumeStatistics.time( - PlumeStatistics.TIME_OPEN_DRIVER, - { GraphDatabase.driver(s"bolt://$hostname:$port", AuthTokens.basic(username, password)) } - ) - private val typeSystem = driver.defaultTypeSystem() + private val logger = LoggerFactory.getLogger(classOf[Neo4jDriver]) + private val connected = new AtomicBoolean(true) + private val driver = GraphDatabase.driver(s"bolt://$hostname:$port", AuthTokens.basic(username, password)) + private val typeSystem = TypeSystem.getDefault override def isConnected: Boolean = connected.get() override def clear(): Unit = Using.resource(driver.session()) { session => - session.writeTransaction { tx => + session.executeWrite { tx => tx.run(""" |MATCH (n) |DETACH DELETE n @@ -50,17 +44,13 @@ final class Neo4jDriver( } } - override def close(): Unit = PlumeStatistics.time( - PlumeStatistics.TIME_CLOSE_DRIVER, { - Try(driver.close()) match { - case Failure(e) => logger.warn("Exception thrown while attempting to close graph.", e) - case Success(_) => connected.set(false) - } - } - ) + override def close(): Unit = Try(driver.close()) match { + case Failure(e) => logger.warn("Exception thrown while attempting to close graph.", e) + case Success(_) => connected.set(false) + } override def exists(nodeId: Long): Boolean = Using.resource(driver.session()) { session => - session.writeTransaction { tx => + session.executeRead { tx => CollectionHasAsScala( tx .run( @@ -80,7 +70,7 @@ final class Neo4jDriver( override def exists(srcId: Long, dstId: Long, edge: String): Boolean = Using.resource(driver.session()) { session => - session.writeTransaction { tx => + session.executeRead { tx => tx .run( s""" diff --git a/drivers/neptune/src/main/scala/com/github/plume/oss/drivers/NeptuneDriver.scala b/drivers/neptune/src/main/scala/com/github/plume/oss/drivers/NeptuneDriver.scala index c76c58db..925613e9 100644 --- a/drivers/neptune/src/main/scala/com/github/plume/oss/drivers/NeptuneDriver.scala +++ b/drivers/neptune/src/main/scala/com/github/plume/oss/drivers/NeptuneDriver.scala @@ -1,6 +1,5 @@ package com.github.plume.oss.drivers -import com.github.plume.oss.PlumeStatistics import com.github.plume.oss.drivers.NeptuneDriver.DEFAULT_PORT import io.circe.Decoder import io.circe.generic.semiauto.deriveDecoder @@ -41,20 +40,17 @@ final class NeptuneDriver( private var cluster = connectToCluster - private def connectToCluster = PlumeStatistics.time( - PlumeStatistics.TIME_OPEN_DRIVER, { - Cluster - .build() - .addContactPoints(hostname) - .port(port) - .enableSsl(true) - .maxInProcessPerConnection(32) - .maxSimultaneousUsagePerConnection(32) - .serializer(Serializers.GRAPHBINARY_V1D0) - .keyCertChainFile(keyCertChainFile) - .create() - } - ) + private def connectToCluster = + Cluster + .build() + .addContactPoints(hostname) + .port(port) + .enableSsl(true) + .maxInProcessPerConnection(32) + .maxSimultaneousUsagePerConnection(32) + .serializer(Serializers.GRAPHBINARY_V1D0) + .keyCertChainFile(keyCertChainFile) + .create() override def g(): GraphTraversalSource = { traversalSource match { @@ -142,18 +138,14 @@ final class NeptuneDriver( } } - override def close(): Unit = PlumeStatistics.time( - PlumeStatistics.TIME_CLOSE_DRIVER, { - try { - cluster.close() - } catch { - case e: Exception => logger.error("Exception thrown while attempting to close graph.", e) - } finally { - traversalSource = None - backend.close() - } - } - ) + override def close(): Unit = try { + cluster.close() + } catch { + case e: Exception => logger.error("Exception thrown while attempting to close graph.", e) + } finally { + traversalSource = None + backend.close() + } } diff --git a/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala b/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala index c950c483..47701161 100644 --- a/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala +++ b/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala @@ -1,9 +1,9 @@ package com.github.plume.oss.drivers -import com.github.plume.oss.PlumeStatistics import com.github.plume.oss.drivers.OverflowDbDriver.newOverflowGraph import com.github.plume.oss.util.BatchedUpdateUtil import com.github.plume.oss.util.BatchedUpdateUtil.* +import io.shiftleft.codepropertygraph.cpgloading.CpgLoader import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* import org.apache.commons.text.StringEscapeUtils @@ -25,7 +25,7 @@ import scala.util.* * @param serializationStatsEnabled * enables saving of serialization statistics. */ -final case class OverflowDbDriver( +final class OverflowDbDriver( storageLocation: Option[String] = Option(JFile.createTempFile("plume-", ".odb").getAbsolutePath), heapPercentageThreshold: Int = 80, serializationStatsEnabled: Boolean = false @@ -44,20 +44,17 @@ final case class OverflowDbDriver( /** A direct pointer to the code property graph object. */ - val cpg: Cpg = - PlumeStatistics.time(PlumeStatistics.TIME_OPEN_DRIVER, { newOverflowGraph(odbConfig) }) + val cpg: Cpg = newOverflowGraph(odbConfig) + + CpgLoader.createIndexes(cpg) override def isConnected: Boolean = !cpg.graph.isClosed - override def close(): Unit = PlumeStatistics.time( - PlumeStatistics.TIME_CLOSE_DRIVER, { - Try(cpg.close()) match { - case Success(_) => - case Failure(e) => - logger.warn("Exception thrown while attempting to close graph.", e) - } - } - ) + override def close(): Unit = Try(cpg.close()) match { + case Success(_) => + case Failure(e) => + logger.warn("Exception thrown while attempting to close graph.", e) + } override def clear(): Unit = { cpg.graph.nodes.asScala.foreach(safeRemove) @@ -69,27 +66,7 @@ final case class OverflowDbDriver( cpg.graph.node(srcId).out(edge).asScala.exists { dst => dst.id() == dstId } override def bulkTx(dg: DiffOrBuilder): Int = { - // Do node operations first - dg.iterator.asScala.foreach { - case node: DetachedNodeData => - val nodeId = node.pID - node.setRefOrId(nodeId) - val newNode = cpg.graph.addNode(nodeId, node.label) - BatchedUpdateUtil.propertiesFromNodeData(node).foreach { case (k, v) => newNode.setProperty(k, v) } - case change: BatchedUpdate.SetNodeProperty => - cpg.graph.node(change.node.id()).setProperty(change.label, change.value) - case _ => // do nothing - } - // Now that all nodes are in, connect edges - dg.iterator.asScala.foreach { - case change: BatchedUpdate.CreateEdge => - val srcId: Long = change.src.pID - val dstId: Long = change.dst.pID - val e: overflowdb.Edge = - cpg.graph.node(srcId).addEdge(change.label, cpg.graph.node(dstId)) - unpack(change.propertiesAndKeys).foreach { case (k: String, v: Any) => e.setProperty(k, v) } - case _ => // do nothing - } + BatchedUpdate.applyDiff(cpg.graph, dg) dg.size() } diff --git a/drivers/overflowdb/src/test/scala/com/github/plume/oss/drivers/OverflowDbTests.scala b/drivers/overflowdb/src/test/scala/com/github/plume/oss/drivers/OverflowDbTests.scala index 62e7f829..e9e90fba 100644 --- a/drivers/overflowdb/src/test/scala/com/github/plume/oss/drivers/OverflowDbTests.scala +++ b/drivers/overflowdb/src/test/scala/com/github/plume/oss/drivers/OverflowDbTests.scala @@ -2,12 +2,11 @@ package com.github.plume.oss.drivers import com.github.plume.oss.testfixtures.PlumeDriverFixture import com.github.plume.oss.testfixtures.PlumeDriverFixture.{b1, m1} -import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} -import io.shiftleft.passes.IntervalKeyPool +import io.shiftleft.codepropertygraph.generated.EdgeTypes import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph import overflowdb.BatchedUpdate -import java.io.{File => JFile} +import java.io.File as JFile import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import scala.util.{Failure, Try} @@ -25,11 +24,11 @@ class OverflowDbTests extends PlumeDriverFixture(new OverflowDbDriver()) { val outFile = Paths.get("./odbGraph.xml").toFile td.exportAsGraphML(outFile) // Should be valid if TinkerGraph can accept it - Try({ + Try { val graph = TinkerGraph.open() graph.traversal().io[Any](outFile.getAbsolutePath).read().iterate() graph.close() - }) match { + } match { case Failure(e) => fail("TinkerGraph could not import ODB generated XML", e) case _ => } @@ -38,7 +37,7 @@ class OverflowDbTests extends PlumeDriverFixture(new OverflowDbDriver()) { private def createSimpleGraph(driver: IDriver): Unit = { val diffGraph = new BatchedUpdate.DiffGraphBuilder() - diffGraph.addNode(m1).addNode(b1).addEdge(m1, b1, EdgeTypes.AST) + diffGraph.addNode(m1.copy).addNode(b1.copy).addEdge(m1.copy, b1.copy, EdgeTypes.AST) driver.bulkTx(diffGraph) } diff --git a/drivers/tinkergraph/src/main/scala/com/github/plume/oss/drivers/TinkerGraphDriver.scala b/drivers/tinkergraph/src/main/scala/com/github/plume/oss/drivers/TinkerGraphDriver.scala index 661d990d..ecff7272 100644 --- a/drivers/tinkergraph/src/main/scala/com/github/plume/oss/drivers/TinkerGraphDriver.scala +++ b/drivers/tinkergraph/src/main/scala/com/github/plume/oss/drivers/TinkerGraphDriver.scala @@ -1,6 +1,5 @@ package com.github.plume.oss.drivers -import com.github.plume.oss.PlumeStatistics import org.slf4j.{Logger, LoggerFactory} import java.io.File @@ -36,13 +35,9 @@ final class TinkerGraphDriver extends GremlinDriver { if (!isSupportedExtension(filePath)) { throw new RuntimeException("Unsupported graph extension! Supported types are GraphML, GraphSON, and Gryo.") } - PlumeStatistics.time( - PlumeStatistics.TIME_CLOSE_DRIVER, { - Using.resource(this.graph.traversal()) { g => - g.io[Any](filePath).write().iterate() - } - } - ) + Using.resource(this.graph.traversal()) { g => + g.io[Any](filePath).write().iterate() + } } /** Imports a .xml, .json, or .kryo TinkerGraph file into the currently connected graph. @@ -60,13 +55,9 @@ final class TinkerGraphDriver extends GremlinDriver { if (!new File(filePath).exists) { throw new RuntimeException(s"No existing serialized graph file was found at $filePath") } - PlumeStatistics.time( - PlumeStatistics.TIME_OPEN_DRIVER, { - Using.resource(this.graph.traversal()) { g => - g.io[Any](filePath).read().iterate() - } - } - ) + Using.resource(this.graph.traversal()) { g => + g.io[Any](filePath).read().iterate() + } } /** Determines if the extension of the given file path is supported by TinkerGraph I/O. diff --git a/project/Projects.scala b/project/Projects.scala index 60336772..93ece233 100644 --- a/project/Projects.scala +++ b/project/Projects.scala @@ -4,15 +4,15 @@ object Projects { val driversRoot = file("drivers") - lazy val base = project.in(driversRoot / "base") + lazy val base = project.in(driversRoot / "base") lazy val neo4j = project.in(driversRoot / "neo4j") - lazy val neo4jEmbed = project.in(driversRoot / "neo4j-embedded") + lazy val neo4jEmbed = project.in(driversRoot / "neo4j-embedded") lazy val tigergraph = project.in(driversRoot / "tigergraph") lazy val gremlin = project.in(driversRoot / "gremlin") lazy val neptune = project.in(driversRoot / "neptune") lazy val overflowdb = project.in(driversRoot / "overflowdb") lazy val tinkergraph = project.in(driversRoot / "tinkergraph") - lazy val commons = project.in(file("commons")) - + lazy val commons = project.in(file("commons")) + lazy val astcreator = project.in(file("astcreator")) } diff --git a/project/Versions.scala b/project/Versions.scala index 8dfd9ae4..ff45368d 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -2,13 +2,13 @@ object Versions { // SAST val codePropertyGraph = "1.6.14" - val joern = "2.0.397" + val joern = "2.0.421" // Drivers val tinkerGraph = "3.4.11" val neo4j = "5.20.0" val overflowDb = "1.192" - val flatGraph = "0.0.59" + val flatGraph = "0.0.70" // Utilities val apacheCodec = "1.15" @@ -17,11 +17,12 @@ object Versions { val apacheCommonsText = "1.12.0" val sttp = "3.9.0" val jackson = "2.13.2" + val jmh = "1.37" val lz4 = "1.8.0" - val slf4j = "2.0.5" - val log4j = "2.20.0" + val slf4j = "2.0.12" + val log4j = "2.23.1" val logback = "1.2.11" - val scalatest = "3.2.15" + val scalatest = "3.2.18" val circe = "0.14.1" } diff --git a/project/build.properties b/project/build.properties index 303541e5..be54e776 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.9.6 +sbt.version = 1.10.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index f86809fd..08e589b8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,4 @@ -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") diff --git a/src/main/java/org/cache2k/benchmark/jmh/ForcedGcMemoryProfiler.java b/src/main/java/org/cache2k/benchmark/jmh/ForcedGcMemoryProfiler.java new file mode 100644 index 00000000..b20d5928 --- /dev/null +++ b/src/main/java/org/cache2k/benchmark/jmh/ForcedGcMemoryProfiler.java @@ -0,0 +1,310 @@ +package org.cache2k.benchmark.jmh; + +/* + * #%L + * Benchmarks: JMH suite. + * %% + * Copyright (C) 2013 - 2021 headissue GmbH, Munich + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.Aggregator; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.runner.IterationType; +import org.openjdk.jmh.util.ListStatistics; +import org.openjdk.jmh.util.Utils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.PrintStream; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Record the used heap memory of a benchmark iteration by forcing a full garbage collection. + * Experimental, not recommended for usage. Use {@link HeapProfiler} instead + * + * @author Jens Wilke + */ +public class ForcedGcMemoryProfiler implements InternalProfiler { + + private static boolean runOnlyAfterLastIteration = true; + @SuppressWarnings("unused") + private static Object keepReference; + private static long gcTimeMillis = -1; + private static long usedHeapViaHistogram = -1; + private static volatile boolean enabled = false; + private static UsageTuple usageAfterIteration; + private static UsageTuple usageAfterSettled; + + /** + * The benchmark needs to hand over the reference so the memory is kept after + * the shutdown of the benchmark and can be measured. + */ + public static void keepReference(Object _rootReferenceToKeep) { + if (enabled) { + keepReference = _rootReferenceToKeep; + } + } + + public static UsageTuple getUsage() { + MemoryUsage _heapUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + MemoryUsage _nonHeapUsage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); + long _usedHeapMemory = _heapUsage.getUsed(); + long _usedNonHeap = _nonHeapUsage.getUsed(); + System.err.println("[getMemoryMXBean] usedHeap=" + _usedHeapMemory + ", usedNonHeap=" + _usedNonHeap + ", totalUsed=" + (_usedHeapMemory + _usedNonHeap)); + System.err.println("[Runtime totalMemory-freeMemory] used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); + return new UsageTuple(_heapUsage, _nonHeapUsage); + } + + /** + * Called from the benchmark when the objects are still referenced to record the + * used memory. Enforces a full garbage collection and records memory usage. + * Waits and triggers GC again, as long as the memory is still reducing. Some workloads + * needs some time until they drain queues and finish all the work. + */ + public static void recordUsedMemory() { + long t0 = System.currentTimeMillis(); + long usedMemorySettled; + if (runSystemGC()) { + usageAfterIteration = getUsage(); + long m2 = usageAfterIteration.getTotalUsed(); + do { + try { + Thread.sleep(567); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + runSystemGC(); + usedMemorySettled = m2; + usageAfterSettled = getUsage(); + m2 = usageAfterSettled.getTotalUsed(); + } while (m2 < usedMemorySettled); + gcTimeMillis = System.currentTimeMillis() - t0; + } + usedHeapViaHistogram = printHeapHistogram(System.out, 30); + } + + public static boolean runSystemGC() { + List enabledBeans = new ArrayList<>(); + + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + long count = bean.getCollectionCount(); + if (count != -1) { + enabledBeans.add(bean); + } + } + + long beforeGcCount = countGc(enabledBeans); + + System.runFinalization(); + System.gc(); + System.runFinalization(); + System.gc(); + + final int MAX_WAIT_MSECS = 20 * 1000; + final int STABLE_TIME_MSECS = 500; + + if (enabledBeans.isEmpty()) { + System.err.println("WARNING: MXBeans can not report GC info."); + return false; + } + + boolean gcHappened = false; + + long start = System.nanoTime(); + long gcHappenedTime = 0; + while (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < MAX_WAIT_MSECS) { + try { + TimeUnit.MILLISECONDS.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long afterGcCount = countGc(enabledBeans); + + if (!gcHappened) { + if (afterGcCount - beforeGcCount >= 2) { + gcHappened = true; + } + } + if (gcHappened) { + if (afterGcCount == beforeGcCount) { + if (TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - gcHappenedTime) > STABLE_TIME_MSECS) { + return true; + } + } else { + gcHappenedTime = System.nanoTime(); + beforeGcCount = afterGcCount; + } + } + } + if (gcHappened) { + System.err.println("WARNING: System.gc() was invoked but unable to wait while GC stopped, is GC too asynchronous?"); + } else { + System.err.println("WARNING: System.gc() was invoked but couldn't detect a GC occurring, is System.gc() disabled?"); + } + return false; + } + + private static long countGc(final List _enabledBeans) { + long cnt = 0; + for (GarbageCollectorMXBean bean : _enabledBeans) { + cnt += bean.getCollectionCount(); + } + return cnt; + } + + public static String getJmapExcutable() { + String javaHome = System.getProperty("java.home"); + String jreDir = File.separator + "jre"; + if (javaHome.endsWith(jreDir)) { + javaHome = javaHome.substring(0, javaHome.length() - jreDir.length()); + } + return (javaHome + + File.separator + + "bin" + + File.separator + + "jmap" + + (Utils.isWindows() ? ".exe" : "")); + } + + public static long printHeapHistogram(PrintStream out, int _maxLines) { + long _totalBytes = 0; + boolean _partial = false; + try { + Process proc = Runtime.getRuntime().exec(new String[]{ + getJmapExcutable(), + "-histo", + Long.toString(Utils.getPid())}); + InputStream in = proc.getInputStream(); + LineNumberReader r = new LineNumberReader(new InputStreamReader(in)); + String s; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(buffer); + while ((s = r.readLine()) != null) { + if ( s.startsWith("Total")) { + ps.println(s); + String[] sa = s.split("\\s+"); + _totalBytes = Long.parseLong(sa[2]); + } else if (r.getLineNumber() <= _maxLines) { + ps.println(s); + } else { + if (!_partial) { + ps.println("[ ... truncated ... ]"); + } + _partial = true; + } + } + r.close(); + in.close(); + ps.close(); + byte[] _histoOuptut = buffer.toByteArray(); + buffer = new ByteArrayOutputStream(); + ps = new PrintStream(buffer); + ps.println("[Heap Histogram Live Objects] used=" + _totalBytes); + ps.write(_histoOuptut); + ps.println(); + ps.close(); + out.write(buffer.toByteArray()); + } catch (Exception ex) { + System.err.println("ForcedGcMemoryProfiler: error attaching / reading histogram"); + ex.printStackTrace(); + } + return _totalBytes; + } + + int iterationNumber = 0; + + @Override + public Collection afterIteration(final BenchmarkParams benchmarkParams, final IterationParams iterationParams, final IterationResult result) { + if (runOnlyAfterLastIteration) { + if (iterationParams.getType() != IterationType.MEASUREMENT + || iterationParams.getCount() != ++iterationNumber) { + return Collections.emptyList(); + } + } + recordUsedMemory(); + List l = new ArrayList<>(); + l.addAll(Arrays.asList( + new OptionalScalarResult("+forced-gc-mem.gcTimeMillis", (double) gcTimeMillis, "ms", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.usedHeap", (double) usedHeapViaHistogram, "bytes", AggregationPolicy.AVG) + )); + if (usageAfterIteration != null) { + // old metrics, t.b. removed + l.addAll(Arrays.asList( + new OptionalScalarResult("+forced-gc-mem.used.settled", (double) usageAfterSettled.getTotalUsed(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.used.after", (double) usageAfterIteration.getTotalUsed(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.total", (double) usageAfterSettled.getTotalCommitted(), "bytes", AggregationPolicy.AVG) + )); + l.addAll(Arrays.asList( + new OptionalScalarResult("+forced-gc-mem.totalUsed", (double) usageAfterSettled.getTotalUsed(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.totalUsed.after", (double) usageAfterIteration.getTotalUsed(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.totalCommitted", (double) usageAfterSettled.getTotalCommitted(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.totalCommitted.after", (double) usageAfterIteration.getTotalCommitted(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.heapUsed", (double) usageAfterSettled.heap.getUsed(), "bytes", AggregationPolicy.AVG), + new OptionalScalarResult("+forced-gc-mem.heapUsed.after", (double) usageAfterIteration.heap.getUsed(), "bytes", AggregationPolicy.AVG) + )); + } + LinuxVmProfiler.addLinuxVmStats("+forced-gc-mem.linuxVm", l); + keepReference = null; + return l; + } + + @Override + public void beforeIteration(final BenchmarkParams benchmarkParams, final IterationParams iterationParams) { + usageAfterIteration = usageAfterSettled = null; + enabled = true; + } + + @Override + public String getDescription() { + return "Adds used memory to the result, if recorded via recordUsedMemory()"; + } + + static class UsageTuple { + MemoryUsage heap; + MemoryUsage nonHeap; + + public UsageTuple(final MemoryUsage _heapUsage, final MemoryUsage _nonHeapUsage) { + heap = _heapUsage; nonHeap = _nonHeapUsage; + } + + public long getTotalUsed() { + return heap.getUsed() + nonHeap.getUsed(); + } + + public long getTotalCommitted() { + return heap.getCommitted() + nonHeap.getCommitted(); + } + + } + +} \ No newline at end of file diff --git a/src/main/java/org/cache2k/benchmark/jmh/GcProfiler.java b/src/main/java/org/cache2k/benchmark/jmh/GcProfiler.java new file mode 100644 index 00000000..0c0c2fd8 --- /dev/null +++ b/src/main/java/org/cache2k/benchmark/jmh/GcProfiler.java @@ -0,0 +1,383 @@ +package org.cache2k.benchmark.jmh; + +/* + * #%L + * Benchmarks: JMH suite. + * %% + * Copyright (C) 2013 - 2021 headissue GmbH, Munich + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +/* + * Copied from JMH GCProfiler. Original copyright: + * + * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.profile.ProfilerException; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ScalarResult; +import org.openjdk.jmh.util.HashMultiset; +import org.openjdk.jmh.util.Multiset; + +import javax.management.ListenerNotFoundException; +import javax.management.Notification; +import javax.management.NotificationEmitter; +import javax.management.NotificationListener; +import javax.management.openmbean.CompositeData; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class GcProfiler implements InternalProfiler { + + private long beforeTime; + private long beforeGCCount; + private long beforeGCTime; + private HotspotAllocationSnapshot beforeAllocated; + private final NotificationListener listener; + private volatile Multiset churn; + private List usedAfterGc = Collections.synchronizedList(new ArrayList<>()); + private List committedAfterGc = Collections.synchronizedList(new ArrayList<>()); + + public GcProfiler() throws ProfilerException { + churn = new HashMultiset(); + + NotificationListener listener; + try { + final Class infoKlass = Class.forName("com.sun.management.GarbageCollectionNotificationInfo"); + final Field notifNameField = infoKlass.getField("GARBAGE_COLLECTION_NOTIFICATION"); + final Method infoMethod = infoKlass.getMethod("from", CompositeData.class); + final Method getGcInfo = infoKlass.getMethod("getGcInfo"); + final Method getMemoryUsageBeforeGc = getGcInfo.getReturnType().getMethod("getMemoryUsageBeforeGc"); + final Method getMemoryUsageAfterGc = getGcInfo.getReturnType().getMethod("getMemoryUsageAfterGc"); + + listener = new NotificationListener() { + @Override + public void handleNotification(Notification n, Object o) { + try { + if (n.getType().equals(notifNameField.get(null))) { + StringBuilder debugLine = new StringBuilder(); + Object info = infoMethod.invoke(null, n.getUserData()); + Object gcInfo = getGcInfo.invoke(info); + Map mapBefore = (Map) getMemoryUsageBeforeGc.invoke(gcInfo); + Map mapAfter = (Map) getMemoryUsageAfterGc.invoke(gcInfo); + long committed = 0; + long used = 0; + for (Map.Entry entry : mapAfter.entrySet()) { + String name = entry.getKey(); + MemoryUsage after = entry.getValue(); + committed += after.getCommitted(); + used += after.getUsed(); + debugLine.append(name).append("=").append(after.getUsed()).append(", "); + MemoryUsage before = mapBefore.get(name); + long c = before.getUsed() - after.getUsed(); + if (c > 0) { + churn.add(name, c); + } + } + usedAfterGc.add(used); + committedAfterGc.add(committed); + System.out.println("[GC Notification Listener] " + debugLine + "Total used=" + used + ", Total committed=" + committed); + } + } catch (IllegalAccessException e) { + } catch (InvocationTargetException e) { + } + } + }; + } catch (ClassNotFoundException e) { + throw new ProfilerException(e); + } catch (NoSuchFieldException e) { + throw new ProfilerException(e); + } catch (NoSuchMethodException e) { + throw new ProfilerException(e); + } + + this.listener = listener; + } + + @Override + public String getDescription() { + return "GC profiling via standard MBeans"; + } + + @Override + public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { + installHooks(); + + long gcTime = 0; + long gcCount = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += bean.getCollectionCount(); + gcTime += bean.getCollectionTime(); + } + this.beforeGCCount = gcCount; + this.beforeGCTime = gcTime; + this.beforeAllocated = HotspotAllocationSnapshot.create(); + this.beforeTime = System.nanoTime(); + } + + @Override + public Collection afterIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams, IterationResult iResult) { + + try { + Thread.sleep(567); + } catch (InterruptedException ignore) { + + } + + uninstallHooks(); + long afterTime = System.nanoTime(); + + HotspotAllocationSnapshot newSnapshot = HotspotAllocationSnapshot.create(); + + long gcTime = 0; + long gcCount = 0; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += bean.getCollectionCount(); + gcTime += bean.getCollectionTime(); + } + + List> results = new ArrayList<>(); + + if (beforeAllocated == HotspotAllocationSnapshot.EMPTY) { + results.add(new ScalarResult(Defaults.PREFIX + "gc.alloc.rate", + Double.NaN, + "MB/sec", AggregationPolicy.AVG)); + } else { + long allocated = newSnapshot.subtract(beforeAllocated); + results.add(new ScalarResult(Defaults.PREFIX + "gc.alloc.rate", + (afterTime != beforeTime) ? + 1.0 * allocated / 1024 / 1024 * TimeUnit.SECONDS.toNanos(1) / (afterTime - beforeTime) : + Double.NaN, + "MB/sec", AggregationPolicy.AVG)); + if (allocated != 0) { + long allOps = iResult.getMetadata().getAllOps(); + results.add(new ScalarResult(Defaults.PREFIX + "gc.alloc.rate.norm", + (allOps != 0) ? + 1.0 * allocated / allOps : + Double.NaN, + "B/op", AggregationPolicy.AVG)); + } + } + + results.add(new ScalarResult( + Defaults.PREFIX + "gc.count", + gcCount - beforeGCCount, + "counts", + AggregationPolicy.AVG)); + + if (gcCount != beforeGCCount || gcTime != beforeGCTime) { + results.add(new ScalarResult( + Defaults.PREFIX + "gc.time", + gcTime - beforeGCTime, + "ms", + AggregationPolicy.AVG)); + } + + for (String space : churn.keys()) { + double churnRate = (afterTime != beforeTime) ? + 1.0 * churn.count(space) * TimeUnit.SECONDS.toNanos(1) / (afterTime - beforeTime) / 1024 / 1024 : + Double.NaN; + + double churnNorm = 1.0 * churn.count(space) / iResult.getMetadata().getAllOps(); + + String spaceName = space.replaceAll(" ", "_"); + + results.add(new ScalarResult( + Defaults.PREFIX + "gc.churn." + spaceName + "", + churnRate, + "MB/sec", + AggregationPolicy.AVG)); + + results.add(new ScalarResult(Defaults.PREFIX + "gc.churn." + spaceName + ".norm", + churnNorm, + "B/op", + AggregationPolicy.AVG)); + } + + if (!usedAfterGc.isEmpty()) { + Collections.sort(usedAfterGc); + long _maximumUsedAfterGc = usedAfterGc.get(usedAfterGc.size() - 1); + results.add(new ScalarResult(Defaults.PREFIX + "gc.maximumUsedAfterGc", + _maximumUsedAfterGc, + "bytes", + AggregationPolicy.AVG)); + System.out.println("maximumUsedAfterGc=" + _maximumUsedAfterGc); + } + if (!committedAfterGc.isEmpty()) { + Collections.sort(committedAfterGc); + long _committedUsedAfterGc = committedAfterGc.get(committedAfterGc.size() - 1); + results.add(new ScalarResult(Defaults.PREFIX + "gc.maximumCommittedAfterGc", + _committedUsedAfterGc, + "bytes", + AggregationPolicy.AVG)); + System.out.println("maximumCommittedAfterGc=" + _committedUsedAfterGc); + } + + return results; + } + + private boolean hooksInstalled; + + public synchronized void installHooks() { + if (hooksInstalled) return; + hooksInstalled = true; + churn = new HashMultiset(); + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + ((NotificationEmitter) bean).addNotificationListener(listener, null, null); + } + } + + public synchronized void uninstallHooks() { + if (!hooksInstalled) return; + hooksInstalled = false; + for (GarbageCollectorMXBean bean : ManagementFactory.getGarbageCollectorMXBeans()) { + try { + ((NotificationEmitter) bean).removeNotificationListener(listener); + } catch (ListenerNotFoundException e) { + } + } + } + + static class HotspotAllocationSnapshot { + public final static HotspotAllocationSnapshot EMPTY = new HotspotAllocationSnapshot(new long[0], new long[0]); + + private static volatile Method GET_THREAD_ALLOCATED_BYTES; + private static volatile boolean allocationNotAvailable; + + private final long[] threadIds; + private final long[] allocatedBytes; + + private HotspotAllocationSnapshot(long[] threadIds, long[] allocatedBytes) { + this.threadIds = threadIds; + this.allocatedBytes = allocatedBytes; + } + + /** + * Takes a snapshot of thread allocation counters. + * The method might allocate, however it is assumed that allocations made by "current thread" will + * be excluded from the result while performing {@link HotspotAllocationSnapshot#subtract(HotspotAllocationSnapshot)} + * + * @return snapshot of thread allocation counters + */ + public static HotspotAllocationSnapshot create() { + Method getBytes = getAllocatedBytesGetter(); + if (getBytes == null) { + return HotspotAllocationSnapshot.EMPTY; + } + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + try { + long[] threadIds = threadMXBean.getAllThreadIds(); + long[] allocatedBytes = (long[]) getBytes.invoke(threadMXBean, (Object) threadIds); + return new HotspotAllocationSnapshot(threadIds, allocatedBytes); + } catch (IllegalAccessException e) { + } catch (InvocationTargetException e) { + } + return HotspotAllocationSnapshot.EMPTY; + } + + /** + * Estimates allocated bytes based on two snapshots. + * The problem is threads can come and go while performing the benchmark, + * thus we would miss allocations made in a thread that was created and died between the snapshots. + *

+ *

Current thread is intentionally excluded since it believed to execute jmh infrastructure code only. + * + * @return estimated number of allocated bytes between profiler calls + */ + public long subtract(HotspotAllocationSnapshot other) { + HashMap prevIndex = new HashMap(); + for (int i = 0; i < other.threadIds.length; i++) { + long id = other.threadIds[i]; + prevIndex.put(id, i); + } + long currentThreadId = Thread.currentThread().getId(); + long allocated = 0; + for (int i = 0; i < threadIds.length; i++) { + long id = threadIds[i]; + if (id == currentThreadId) { + continue; + } + allocated += allocatedBytes[i]; + Integer prev = prevIndex.get(id); + if (prev != null) { + allocated -= other.allocatedBytes[prev]; + } + } + return allocated; + } + + private static Method getAllocatedBytesGetter() { + Method getBytes = GET_THREAD_ALLOCATED_BYTES; + if (getBytes != null || allocationNotAvailable) { + return getBytes; + } + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + try { + getBytes = threadMXBean.getClass().getMethod("getThreadAllocatedBytes", long[].class); + getBytes.setAccessible(true); + } catch (Throwable e) { // To avoid jmh failure in case of incompatible JDK and/or inaccessible method + getBytes = null; + allocationNotAvailable = true; + System.out.println("Allocation profiling is not available: " + e.getMessage()); + } + GET_THREAD_ALLOCATED_BYTES = getBytes; + return getBytes; + } + } + + static class Defaults { + static final String PREFIX = "+c2k."; + } + +} \ No newline at end of file diff --git a/src/main/java/org/cache2k/benchmark/jmh/HeapProfiler.java b/src/main/java/org/cache2k/benchmark/jmh/HeapProfiler.java new file mode 100644 index 00000000..223da6ee --- /dev/null +++ b/src/main/java/org/cache2k/benchmark/jmh/HeapProfiler.java @@ -0,0 +1,147 @@ +package org.cache2k.benchmark.jmh; + +/* + * #%L + * Benchmarks: JMH suite. + * %% + * Copyright (C) 2013 - 2021 headissue GmbH, Munich + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.runner.IterationType; +import org.openjdk.jmh.util.Utils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Record the total size in bytes of active objects on the heap after the last + * benchmark iteration and print a heap histogram. The measure is determined by + * calling {@code jmap} with the PID of the benchmark process. Since calling jmap + * will result in a garbage collection in the VM running the benchmark, this is never + * done in between iterations. + * + * @author Jens Wilke + */ +public class HeapProfiler implements InternalProfiler { + + private static String getJmapExcutable() { + String javaHome = System.getProperty("java.home"); + String jreDir = File.separator + "jre"; + if (javaHome.endsWith(jreDir)) { + javaHome = javaHome.substring(0, javaHome.length() - jreDir.length()); + } + return (javaHome + + File.separator + + "bin" + + File.separator + + "jmap" + + (Utils.isWindows() ? ".exe" : "")); + } + + private static long printHeapHistogram(PrintStream out, int maxLines) { + long totalBytes = 0; + boolean partial = false; + try { + Process proc = Runtime.getRuntime().exec(new String[]{ + getJmapExcutable(), + "-histo:live", + Long.toString(Utils.getPid())}); + InputStream in = proc.getInputStream(); + LineNumberReader r = new LineNumberReader(new InputStreamReader(in)); + String s; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(buffer); + while ((s = r.readLine()) != null) { + if ( s.startsWith("Total")) { + ps.println(s); + String[] sa = s.split("\\s+"); + totalBytes = Long.parseLong(sa[2]); + } else if (r.getLineNumber() <= maxLines) { + ps.println(s); + } else { + if (!partial) { + ps.println("[ ... truncated ... ]"); + } + partial = true; + } + } + r.close(); + in.close(); + ps.close(); + byte[] histoOtput = buffer.toByteArray(); + buffer = new ByteArrayOutputStream(); + ps = new PrintStream(buffer); + ps.println("[jmap heap histogram, truncated at " + maxLines + " lines]"); + ps.write(histoOtput); + ps.println(); + ps.close(); + out.write(buffer.toByteArray()); + } catch (Exception ex) { + System.err.println("ForcedGcMemoryProfiler: error attaching / reading histogram"); + ex.printStackTrace(); + } + return totalBytes; + } + + private static Object reference; + + /** + * Prevent object from being garbage collected at the last iteration. + */ + public static void keepReference(Object target) { + reference = target; + } + + int iterationNumber = 0; + + @Override + public Collection afterIteration(BenchmarkParams benchmarkParams, + IterationParams iterationParams, + IterationResult result) { + if (iterationParams.getType() != IterationType.MEASUREMENT + || iterationParams.getCount() != ++iterationNumber) { + return Collections.emptyList(); + } + long bytes = printHeapHistogram(System.out, 30); + List l = Arrays.asList( + new OptionalScalarResult("+liveObjects", (double) bytes, "bytes", AggregationPolicy.AVG) + ); + return l; + } + + @Override + public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { } + + @Override + public String getDescription() { + return "Adds used bytes of the active heap objects measured at the end of the last iteration."; + } + +} \ No newline at end of file diff --git a/src/main/java/org/cache2k/benchmark/jmh/LinuxVmProfiler.java b/src/main/java/org/cache2k/benchmark/jmh/LinuxVmProfiler.java new file mode 100644 index 00000000..25a2df57 --- /dev/null +++ b/src/main/java/org/cache2k/benchmark/jmh/LinuxVmProfiler.java @@ -0,0 +1,113 @@ +package org.cache2k.benchmark.jmh; + +/* + * #%L + * Benchmarks: JMH suite. + * %% + * Copyright (C) 2013 - 2021 headissue GmbH, Munich + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ScalarResult; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Add Linux VM metrics to the result. Parces {@code /proc/self/status} and adds all + * metrics starting with {@code Vm}. In particular interesting is {@code VmRSS} and {@code VmHWM} + * to analyze the memory usage. + * + * @author Jens Wilke + * @see Blog Article + */ +public class LinuxVmProfiler implements InternalProfiler { + + private static final String PREFIX = "+linux.proc.status"; + + /** + * Parse the linux {@code /proc/self/status} and add everything prefixed with "Vm" as metric to + * the profiling result. + */ + public static void addLinuxVmStats(String prefix, List l) { + File f = new File("/proc/self/status"); + if (!f.exists()) return; + try { + LineNumberReader r = new LineNumberReader(new InputStreamReader(new FileInputStream(f.getAbsolutePath()))); + String line; + while ((line = r.readLine()) != null) { + if (!line.startsWith("Vm")) { + continue; + } + String[] sa = line.split("\\s+"); + if (sa.length != 3) { + throw new IOException("Format error: 3 elements expected"); + } + if (!sa[2].equals("kB")) { + throw new IOException("Format error: unit kB expected, was: " + sa[2]); + } + String name = sa[0].substring(0, sa[0].length() - 1); + l.add( + new OptionalScalarResult(prefix + "." + name, (double) Long.parseLong(sa[1]), "kB", AggregationPolicy.AVG) + ); + } + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + @Override + public Collection afterIteration(BenchmarkParams benchmarkParams, + IterationParams iterationParams, IterationResult result) { + List l = new ArrayList<>(); + addLinuxVmStats(PREFIX, l); + return l; + } + + @Override + public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { + + } + + @Override + public String getDescription() { + return "Adds Linux VM metrics to the result"; + } + +} \ No newline at end of file diff --git a/src/main/java/org/cache2k/benchmark/jmh/OptionalScalarResult.java b/src/main/java/org/cache2k/benchmark/jmh/OptionalScalarResult.java new file mode 100644 index 00000000..57e876bf --- /dev/null +++ b/src/main/java/org/cache2k/benchmark/jmh/OptionalScalarResult.java @@ -0,0 +1,81 @@ +package org.cache2k.benchmark.jmh; + +/* + * #%L + * Benchmarks: JMH suite. + * %% + * Copyright (C) 2013 - 2021 headissue GmbH, Munich + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.Aggregator; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ResultRole; +import org.openjdk.jmh.results.ScalarResult; +import org.openjdk.jmh.util.ListStatistics; +import org.openjdk.jmh.util.Statistics; + +import java.util.Collection; + +/** + * Same as {@link ScalarResult} but don't fill missing values with 0. + */ +public class OptionalScalarResult extends Result { + + public OptionalScalarResult(String label, double n, String unit, AggregationPolicy policy) { + this(label, of(n), unit, policy); + } + + OptionalScalarResult(String label, Statistics s, String unit, AggregationPolicy policy) { + super(ResultRole.SECONDARY, label, s, unit, policy); + } + + @Override + protected Aggregator getThreadAggregator() { + return new MyScalarResultAggregator(); + } + + @Override + protected Aggregator getIterationAggregator() { + return new MyScalarResultAggregator(); + } + + @Override + protected OptionalScalarResult getZeroResult() { + return null; + } + + AggregationPolicy getPolicy() { return policy; } + + static class MyScalarResultAggregator implements Aggregator { + + @Override + public OptionalScalarResult aggregate(Collection results) { + ListStatistics stats = new ListStatistics(); + for (OptionalScalarResult r : results) { + stats.addValue(r.getScore()); + } + OptionalScalarResult first = results.iterator().next(); + return new OptionalScalarResult( + first.getLabel(), + stats, + first.getScoreUnit(), + first.getPolicy() + ); + } + } + +} \ No newline at end of file diff --git a/src/main/scala/com/github/plume/oss/Benchmark.scala b/src/main/scala/com/github/plume/oss/Benchmark.scala new file mode 100644 index 00000000..a2ff671d --- /dev/null +++ b/src/main/scala/com/github/plume/oss/Benchmark.scala @@ -0,0 +1,162 @@ +package com.github.plume.oss + +import com.github.plume.oss.Benchmark.BenchmarkType.WRITE +import com.github.plume.oss.drivers.IDriver +import io.joern.jimple2cpg.Config +import org.cache2k.benchmark.jmh.ForcedGcMemoryProfiler +import org.openjdk.jmh.annotations.{Benchmark, Level, Mode, Param, Scope, Setup, State, TearDown} +import org.openjdk.jmh.infra.{BenchmarkParams, Blackhole} +import org.openjdk.jmh.runner.Runner +import org.openjdk.jmh.runner.options.{ChainedOptionsBuilder, OptionsBuilder, TimeValue} +import upickle.default.* + +import java.util.concurrent.TimeUnit +import scala.compiletime.uninitialized + +object Benchmark { + + def main(args: Array[String]): Unit = { + Plume + .optionParser("plume-benchmark", "A benchmarking suite for graph databases as static analysis backends.") + .parse(args, PlumeConfig()) + .foreach { config => + val writeOptsBenchmark = createOptionsBoilerPlate(config, WRITE) + .include(classOf[GraphWriteBenchmark].getSimpleName) + .build() + new Runner(writeOptsBenchmark).run() + println( + s"Finished WRITE JMH benchmarks. Results: ${config.jmhResultFile}-WRITE.csv; Output: ${config.jmhOutputFile}-WRITE.csv" + ) + +// val readOptsBenchmark = createOptionsBoilerPlate(config, READ) +// .include(classOf[OverflowDbBenchmark].getSimpleName) +// .build() +// new Runner(readOptsBenchmark).run() +// println( +// s"Finished READ JMH benchmarks. Results: ${config.jmhResultFile}-READ.csv; Output: ${config.jmhOutputFile}-READ.csv" +// ) + } + } + + private def createOptionsBoilerPlate(config: PlumeConfig, benchmarkType: BenchmarkType): ChainedOptionsBuilder = { + new OptionsBuilder() + .addProfiler(classOf[ForcedGcMemoryProfiler]) + .warmupIterations(1) + .warmupTime(TimeValue.seconds(1)) + .measurementTime(TimeValue.seconds(2)) + .measurementIterations(3) + .mode(Mode.AverageTime) + .timeUnit(TimeUnit.NANOSECONDS) + .forks(2) + .output(s"${config.jmhOutputFile}-$benchmarkType.txt") + .result(s"${config.jmhResultFile}-$benchmarkType.csv") + .param("configStr", write(config)) + .detectJvmArgs() // inherit stuff like max heap size + } + + enum BenchmarkType { + case READ, WRITE + } + +} + +@State(Scope.Benchmark) +class GraphWriteBenchmark { + + @Param(Array("")) + var configStr: String = "" + var config: PlumeConfig = + if (!configStr.isBlank) read[PlumeConfig](configStr) else PlumeConfig() + var driver: IDriver = uninitialized + + @Setup + def setupBenchmark(params: BenchmarkParams): Unit = { + config = if (!configStr.isBlank) read[PlumeConfig](configStr) else PlumeConfig() + driver = config.dbConfig.toDriver + } + + @Setup(Level.Iteration) + def clearDriver(params: BenchmarkParams): Unit = { + driver.clear() + } + + @Benchmark + def createAst(blackhole: Blackhole): Unit = { + JimpleAst2Database(driver).createAst(Config().withInputPath(config.inputDir)) + Option(blackhole).foreach(_.consume(driver)) + } + + @TearDown + def cleanupBenchmark(): Unit = { + driver.clear() + driver.close() + } + +} + +sealed trait GraphReadBenchmark[D <: IDriver](protected val driver: D) { + + private var nodeStart: Array[Long] = new Array[Long](0) + private var fullNames: Array[String] = uninitialized + + @Setup + def setupFun(params: BenchmarkParams): Unit = { + params.getBenchmark + } + + @Benchmark + def astDFS(blackhole: Blackhole): Int + + @Benchmark + def astUp(blackhole: Blackhole): Int + + @Benchmark + def orderSumChecked(blackhole: Blackhole): Int + + @Benchmark + def orderSumUnchecked(blackhole: Blackhole): Int + + @Benchmark + def orderSumExplicit(blackhole: Blackhole): Int + + @Benchmark + def callOrderTrav(blackhole: Blackhole): Int + + @Benchmark + def callOrderExplicit(blackhole: Blackhole): Int + + @Benchmark + def indexedMethodFullName(bh: Blackhole): Unit + + @Benchmark + def unindexedMethodFullName(bh: Blackhole): Unit + +} + +//@State(Scope.Benchmark) +//class OverflowDbBenchmark(config: OverflowDbConfig) +// extends GraphReadBenchmark( +// ) { +// +// override def createAst(blackhole: Blackhole): Int = { +// 0 +// } +// +// override def astDFS(blackhole: Blackhole): Int = ??? +// +// override def astUp(blackhole: Blackhole): Int = ??? +// +// override def orderSumChecked(blackhole: Blackhole): Int = ??? +// +// override def orderSumUnchecked(blackhole: Blackhole): Int = ??? +// +// override def orderSumExplicit(blackhole: Blackhole): Int = ??? +// +// override def callOrderTrav(blackhole: Blackhole): Int = ??? +// +// override def callOrderExplicit(blackhole: Blackhole): Int = ??? +// +// override def indexedMethodFullName(bh: Blackhole): Unit = ??? +// +// override def unindexedMethodFullName(bh: Blackhole): Unit = ??? +//} diff --git a/src/main/scala/com/github/plume/oss/Plume.scala b/src/main/scala/com/github/plume/oss/Plume.scala index 0459b2dd..66324344 100644 --- a/src/main/scala/com/github/plume/oss/Plume.scala +++ b/src/main/scala/com/github/plume/oss/Plume.scala @@ -1,17 +1,12 @@ package com.github.plume.oss -import better.files.File -import com.github.plume.oss.drivers._ -import io.circe.Json +import com.github.plume.oss.drivers.* import io.joern.jimple2cpg.Config -import io.joern.x2cpg.X2Cpg -import scopt.OParser - -import java.io.InputStreamReader +import scopt.{OParser, OptionParser} /** Entry point for command line CPG creator */ -object Plume extends App { +object Plume { private val frontendSpecificOptions = { val builder = OParser.builder[Config] @@ -19,90 +14,137 @@ object Plume extends App { OParser.sequence(programName("plume")) } - private def parseDriverConfig(): (DriverConfig, IDriver) = { - import io.circe.generic.auto._ - import io.circe.yaml.parser - def interpretConfig(rawConf: Json) = { - rawConf.as[DriverConfig] match { - case Left(_) => (null, new OverflowDbDriver()) - case Right(conf) => (conf, createDriver(conf)) + def main(args: Array[String]): Unit = { + Plume + .optionParser("plume", "An AST creator for comparing graph databases as static analysis backends.") + .parse(args, PlumeConfig()) + .foreach { config => + val driver = config.dbConfig.toDriver + driver match { + case d: TinkerGraphDriver => + config.dbConfig.asInstanceOf[TinkerGraphConfig].importPath.foreach(d.importGraph) + case _ => + } + new JimpleAst2Database(driver).createAst(Config().withInputPath(config.inputDir)) + driver match { + case d: TinkerGraphDriver => + config.dbConfig.asInstanceOf[TinkerGraphConfig].exportPath.foreach(d.exportGraph) + case _ => + } } - } - val f = File("driver.yaml") - if (!f.exists) { - println("No driver.yaml found, defaulting to OverflowDB driver.") - return (null, new OverflowDbDriver()) - } - parser.parse(new InputStreamReader(f.newInputStream)) match { - case Left(_) => (null, new OverflowDbDriver()) - case Right(rawConf) => interpretConfig(rawConf) - } } - private def createDriver(conf: DriverConfig): IDriver = { - conf match { - case _ if conf.database == "OverflowDB" => - new OverflowDbDriver( - storageLocation = Option(conf.params.getOrElse("storageLocation", "cpg.odb")), - heapPercentageThreshold = conf.params.getOrElse("heapPercentageThreshold", "80").toInt, - serializationStatsEnabled = conf.params.getOrElse("serializationStatsEnabled", "false").toBoolean + def optionParser(name: String, description: String): OptionParser[PlumeConfig] = + new OptionParser[PlumeConfig](name) { + + note(description) + help('h', "help") + + arg[String]("input-dir") + .text("The target application to parse.") + .action((x, c) => c.copy(inputDir = x)) + + opt[String]('o', "jmh-output-file") + .text(s"The JMH output file path. Exclude file extensions.") + .hidden() + .action((x, c) => c.copy(jmhOutputFile = x)) + + opt[String]('r', "jmh-result-file") + .text(s"The result file path. Exclude file extensions.") + .hidden() + .action((x, c) => c.copy(jmhResultFile = x)) + + cmd("tinkergraph") + .action((_, c) => c.copy(dbConfig = TinkerGraphConfig())) + .children( + opt[String]("import-path") + .text("The TinkerGraph to import.") + .action((x, c) => + c.copy(dbConfig = c.dbConfig.asInstanceOf[TinkerGraphConfig].copy(importPath = Option(x))) + ), + opt[String]("export-path") + .text("The TinkerGraph export path to serialize the result to.") + .action((x, c) => + c.copy(dbConfig = c.dbConfig.asInstanceOf[TinkerGraphConfig].copy(exportPath = Option(x))) + ) + ) + + cmd("overflowdb") + .action((_, c) => c.copy(dbConfig = OverflowDbConfig())) + .children( + opt[String]("storage-location") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[OverflowDbConfig].copy(storageLocation = x))), + opt[Int]("heap-percentage-threshold") + .action((x, c) => + c.copy(dbConfig = c.dbConfig.asInstanceOf[OverflowDbConfig].copy(heapPercentageThreshold = x)) + ), + opt[Unit]("enable-serialization-stats") + .action((_, c) => + c.copy(dbConfig = c.dbConfig.asInstanceOf[OverflowDbConfig].copy(serializationStatsEnabled = true)) + ) ) - case _ if conf.database == "TinkerGraph" => new TinkerGraphDriver() - case _ if conf.database == "Neo4j" => - new Neo4jDriver( - hostname = conf.params.getOrElse("hostname", "localhost"), - port = conf.params.getOrElse("port", "7687").toInt, - username = conf.params.getOrElse("username", "neo4j"), - password = conf.params.getOrElse("password", "neo4j"), - txMax = conf.params.getOrElse("txMax", "25").toInt + + cmd("neo4j") + .action((_, c) => c.copy(dbConfig = Neo4jConfig())) + .children( + opt[String]("hostname") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jConfig].copy(hostname = x))), + opt[Int]("port") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jConfig].copy(port = x))), + opt[String]("username") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jConfig].copy(username = x))), + opt[String]("password") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jConfig].copy(password = x))), + opt[Int]("tx-max") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jConfig].copy(txMax = x))) ) - case _ if conf.database == "TigerGraph" => - new TigerGraphDriver( - hostname = conf.params.getOrElse("hostname", "localhost"), - restPpPort = conf.params.getOrElse("restPpPort", "7687").toInt, - gsqlPort = conf.params.getOrElse("gsqlPort", "14240").toInt, - username = conf.params.getOrElse("username", "tigergraph"), - password = conf.params.getOrElse("password", "tigergraph"), - timeout = conf.params.getOrElse("timeout", "3000").toInt, - txMax = conf.params.getOrElse("txMax", "25").toInt, - scheme = conf.params.getOrElse("scheme", "http") + + cmd("neo4j-embedded") + .action((_, c) => c.copy(dbConfig = Neo4jEmbeddedConfig())) + .children( + opt[String]("databaseName") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jEmbeddedConfig].copy(databaseName = x))), + opt[String]("databaseDir") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jEmbeddedConfig].copy(databaseDir = x))), + opt[Int]("tx-max") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[Neo4jEmbeddedConfig].copy(txMax = x))) ) - case _ if conf.database == "Neptune" => - new NeptuneDriver( - hostname = conf.params.getOrElse("hostname", "localhost"), - port = conf.params.getOrElse("port", "8182").toInt, - keyCertChainFile = conf.params.getOrElse("keyCertChainFile", "src/main/resources/conf/SFSRootCAC2.pem"), - txMax = conf.params.getOrElse("txMax", "50").toInt + + cmd("tigergraph") + .action((_, c) => c.copy(dbConfig = TigerGraphConfig())) + .children( + opt[String]("hostname") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(hostname = x))), + opt[Int]("restpp-port") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(restPpPort = x))), + opt[Int]("gsql-port") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(gsqlPort = x))), + opt[String]("username") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(username = x))), + opt[String]("password") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(password = x))), + opt[Int]("timeout") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(timeout = x))), + opt[Int]("tx-max") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(txMax = x))), + opt[String]("scheme") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[TigerGraphConfig].copy(scheme = x))) ) - case _ => - println( - "No supported database specified by driver.yaml. Supported databases are: OverflowDB, TinkerGraph, Neo4j, Neptune, and TigerGraph." - ); null - } - } - private val configOption = X2Cpg.parseCommandLine(args, frontendSpecificOptions, Config()) + cmd("neptune") + .action((_, c) => c.copy(dbConfig = NeptuneConfig())) + .children( + opt[String]("hostname") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[NeptuneConfig].copy(hostname = x))), + opt[Int]("port") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[NeptuneConfig].copy(port = x))), + opt[String]("key-cert-chain-file") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[NeptuneConfig].copy(keyCertChainFile = x))), + opt[Int]("tx-max") + .action((x, c) => c.copy(dbConfig = c.dbConfig.asInstanceOf[NeptuneConfig].copy(txMax = x))) + ) - configOption.foreach { config => - val (conf, driver) = parseDriverConfig() - if (driver == null) { - println("Unable to create driver, bailing out...") - System.exit(1) - } - driver match { - case d: TinkerGraphDriver if conf != null => - val importPath = conf.params.get("importPath") - if (importPath.isDefined) d.importGraph(importPath.get) - case _ => } - new JimpleAst2Database(driver).createAst(config) - driver match { - case d: TinkerGraphDriver if conf != null => - val exportPath = conf.params.get("exportPath") - if (exportPath.isDefined) d.exportGraph(exportPath.get) - case _ => - } - } } diff --git a/src/main/scala/com/github/plume/oss/package.scala b/src/main/scala/com/github/plume/oss/package.scala new file mode 100644 index 00000000..689cbb4a --- /dev/null +++ b/src/main/scala/com/github/plume/oss/package.scala @@ -0,0 +1,72 @@ +package com.github.plume + +import better.files.File +import com.github.plume.oss.drivers.* +import upickle.default.* + +package object oss { + + case class PlumeConfig( + inputDir: String = "", + jmhOutputFile: String = File.newTemporaryFile("plume-jmh-output-").pathAsString, + jmhResultFile: String = File.newTemporaryFile("plume-jmh-result-").pathAsString, + dbConfig: DatabaseConfig = OverflowDbConfig() + ) derives ReadWriter + + sealed trait DatabaseConfig derives ReadWriter { + def toDriver: IDriver + } + + case class TinkerGraphConfig(importPath: Option[String] = None, exportPath: Option[String] = None) + extends DatabaseConfig { + override def toDriver: IDriver = new TinkerGraphDriver() + } + + case class OverflowDbConfig( + storageLocation: String = "cpg.bin", + heapPercentageThreshold: Int = 80, + serializationStatsEnabled: Boolean = false + ) extends DatabaseConfig { + override def toDriver: IDriver = + new OverflowDbDriver(Option(storageLocation), heapPercentageThreshold, serializationStatsEnabled) + } + + case class Neo4jConfig( + hostname: String = "localhost", + port: Int = 7687, + username: String = "neo4j", + password: String = "neo4j", + txMax: Int = 25 + ) extends DatabaseConfig { + override def toDriver: IDriver = new Neo4jDriver(hostname, port, username, password, txMax) + } + + case class Neo4jEmbeddedConfig(databaseName: String = "neo4j", databaseDir: String = "neo4j-db", txMax: Int = 25) + extends DatabaseConfig { + override def toDriver: IDriver = new Neo4jEmbeddedDriver(databaseName, File(databaseDir), txMax) + } + + case class TigerGraphConfig( + hostname: String = "localhost", + restPpPort: Int = 7687, + gsqlPort: Int = 14240, + username: String = "tigergraph", + password: String = "tigergraph", + timeout: Int = 3000, + txMax: Int = 25, + scheme: String = "http" + ) extends DatabaseConfig { + override def toDriver: IDriver = + new TigerGraphDriver(hostname, restPpPort, gsqlPort, username, password, timeout, scheme, txMax) + } + + case class NeptuneConfig( + hostname: String = "localhost", + port: Int = 8182, + keyCertChainFile: String = "src/main/resources/conf/SFSRootCAC2.pem", + txMax: Int = 50 + ) extends DatabaseConfig { + override def toDriver: IDriver = new NeptuneDriver(hostname, port, keyCertChainFile, txMax) + } + +} diff --git a/src/test/resources/default.semantics b/src/test/resources/default.semantics deleted file mode 100644 index 1ead5e3f..00000000 --- a/src/test/resources/default.semantics +++ /dev/null @@ -1,36 +0,0 @@ -# ".sizeOf" reduces the FPs -# 1->-1 first parameter mapped to return value (-1) -".addition" 1->-1 2->-1 -".addressOf" 1->-1 -".alloc" 1->-1 2->-1 -".assignment" 2->1 -".assignmentAnd" 2->1 1->1 -".assignmentArithmeticShiftRight" 2->1 1->1 -".assignmentDivision" 2->1 1->1 -".assignmentExponentiation" 2->1 1->1 -".assignmentLogicalShiftRight" 2->1 1->1 -".assignmentMinus" 2->1 1->1 -".assignmentModulo" 2->1 1->1 -".assignmentMultiplication" 2->1 1->1 -".assignmentOr" 2->1 1->1 -".assignmentPlus" 2->1 1->1 -".assignmentShiftLeft" 2->1 1->1 -".assignmentXor" 2->1 1->1 -".computedMemberAccess" 1->-1 -".conditional" 2->-1 3->-1 -".fieldAccess" 1->-1 -".getElementPtr" 1->-1 -".incBy 2->1 3->1 4->1" -".indexAccess" 1->-1 -".indirectComputedMemberAccess" 1->-1 -".indirectFieldAccess" 1->-1 -".indirectIndexAccess" 1->-1 2->-1 -".indirectMemberAccess" 1->-1 -".indirection" 1->-1 -".memberAccess" 1->-1 -".pointerShift" 1->-1 -".postDecrement" 1->1 -".postIncrement" 1->1 -".preDecrement" 1->1 -".preIncrement" 1->1 -".sizeOf" \ No newline at end of file diff --git a/src/test/resources/log4j2.xml b/src/test/resources/log4j2.xml deleted file mode 100644 index 876e9951..00000000 --- a/src/test/resources/log4j2.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/test/resources/project/build.properties b/src/test/resources/project/build.properties deleted file mode 100644 index 19479ba4..00000000 --- a/src/test/resources/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.5.2 diff --git a/src/test/resources/source_support/Bar.java b/src/test/resources/source_support/Bar.java deleted file mode 100644 index 8751a7f4..00000000 --- a/src/test/resources/source_support/Bar.java +++ /dev/null @@ -1,7 +0,0 @@ -public class Bar { - - public static int test(int a) { - return a + 1; - } - -} \ No newline at end of file diff --git a/src/test/resources/source_support/Foo.java b/src/test/resources/source_support/Foo.java deleted file mode 100644 index 100506a9..00000000 --- a/src/test/resources/source_support/Foo.java +++ /dev/null @@ -1,7 +0,0 @@ -public class Foo { - - public static void main(String[] args) { - Bar.test(2); - } - -} \ No newline at end of file diff --git a/src/test/scala/com/github/plume/oss/JavaCompiler.scala b/src/test/scala/com/github/plume/oss/JavaCompiler.scala deleted file mode 100644 index 6b9cbbd9..00000000 --- a/src/test/scala/com/github/plume/oss/JavaCompiler.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.github.plume.oss - -import java.io.File -import java.util.Collections -import javax.tools.{JavaCompiler => Javac, JavaFileObject, StandardLocation, ToolProvider} -import scala.jdk.CollectionConverters - -/** Compiles a given source file. - */ -object JavaCompiler { - - /** Compiles the source code with debugging info. - */ - def compileJava(sourceCodeFiles: File*): Unit = { - val javac = getJavaCompiler - val fileManager = javac.getStandardFileManager(null, null, null) - javac - .getTask( - null, - fileManager, - null, - CollectionConverters.SeqHasAsJava(Seq("-g", "-d") :+ sourceCodeFiles.head.getParent).asJava, - null, - fileManager.getJavaFileObjectsFromFiles(CollectionConverters.SeqHasAsJava(sourceCodeFiles.toList).asJava) - ) - .call() - - fileManager - .list(StandardLocation.CLASS_OUTPUT, "", Collections.singleton(JavaFileObject.Kind.CLASS), false) - .forEach(x => new File(x.toUri).deleteOnExit()) - } - - /** Programmatically obtains the system Java compiler. - */ - def getJavaCompiler: Javac = { - Option(ToolProvider.getSystemJavaCompiler) match { - case Some(javac) => javac - case None => throw new RuntimeException("Unable to find a Java compiler on the system!") - } - } -} diff --git a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala b/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala deleted file mode 100644 index fe001ecf..00000000 --- a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.github.plume.oss.testfixtures - -import com.github.plume.oss.{JimpleAst2Database, PlumeStatistics} -import com.github.plume.oss.drivers.OverflowDbDriver -import com.github.plume.oss.JavaCompiler.compileJava -import io.joern.jimple2cpg.testfixtures.JimpleCodeToCpgFixture -import io.joern.jimple2cpg.Config -import io.joern.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, DefaultTestCpg} -import io.shiftleft.codepropertygraph.Cpg -import org.slf4j.LoggerFactory - -import java.io.{File, PrintWriter} -import java.nio.file.Path -import scala.util.Using - -trait PlumeFrontend(val _driver: Option[OverflowDbDriver]) extends LanguageFrontend { - - private val logger = LoggerFactory.getLogger(classOf[PlumeFrontend]) - override val fileSuffix: String = ".java" - - val driver: OverflowDbDriver = _driver match { - case Some(d) => d - case None => new OverflowDbDriver() - } - - override def execute(sourceCodeFile: File): Cpg = { - PlumeStatistics.reset() - new JimpleAst2Database(driver).createAst(Config().withInputPath(sourceCodeFile.getAbsolutePath)) - logger.info(s"Plume statistics from last test: ${PlumeStatistics.results()}") - Cpg(driver.cpg.graph) - } -} - -class PlumeTestCpg(_driver: Option[OverflowDbDriver]) extends DefaultTestCpg with PlumeFrontend(_driver) { - - override protected def codeDirPreProcessing(rootFile: Path, codeFiles: List[Path]): Unit = { - val sourceFiles = codeFiles.map(_.toFile).filter(_.getName.endsWith(".java")) - if (sourceFiles.nonEmpty) JimpleCodeToCpgFixture.compileJava(rootFile, sourceFiles) - } -} - -class Jimple2CpgFixture(_driver: Option[OverflowDbDriver] = None) extends Code2CpgFixture(() => PlumeTestCpg(_driver)) diff --git a/testprogram/jfreechart-1.0.19-demo.jar b/testprogram/jfreechart-1.0.19-demo.jar new file mode 100644 index 00000000..d01b535e Binary files /dev/null and b/testprogram/jfreechart-1.0.19-demo.jar differ diff --git a/testprogram/progressbar-0.10.1.jar b/testprogram/progressbar-0.10.1.jar new file mode 100644 index 00000000..3586b35a Binary files /dev/null and b/testprogram/progressbar-0.10.1.jar differ