diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98919ed0..24126a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: run: sbt "project neo4j" test # - name: Run TigerGraph tests # run: | -# curl https://dl.tigergraph.com/enterprise-edition/gsql_client/tigergraph-3.9.3-1-gsql_client.jar \ +# curl https://dl.tigergraph.com/enterprise-edition/gsql_client/tigergraph-3.10.1-gsql_client.jar \ # --output gsql_client.jar && # export GSQL_HOME=`pwd`/gsql_client.jar # sbt "project tigergraph" test diff --git a/build.sbt b/build.sbt index 3699caca..89bc07ee 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ inThisBuild( List( organization := "com.github.plume-oss", version := "2.0.0", - scalaVersion := "3.3.1", + scalaVersion := "3.4.1", resolvers ++= Seq( Resolver.mavenLocal, Resolver.mavenCentral, @@ -36,6 +36,7 @@ libraryDependencies ++= Seq( "io.joern" %% "semanticcpg" % Versions.joern, "io.joern" %% "x2cpg" % Versions.joern, "io.joern" %% "jimple2cpg" % Versions.joern, + "io.joern" %% "jimple2cpg" % Versions.joern % Test classifier "tests", "io.joern" %% "x2cpg" % Versions.joern % Test classifier "tests", "org.slf4j" % "slf4j-api" % Versions.slf4j, "org.apache.logging.log4j" % "log4j-core" % Versions.log4j % Test, diff --git a/commons/build.sbt b/commons/build.sbt index 48eb3c7f..e0c21e60 100644 --- a/commons/build.sbt +++ b/commons/build.sbt @@ -2,6 +2,5 @@ name := "commons" libraryDependencies ++= Seq( "io.shiftleft" %% "overflowdb-traversal" % Versions.overflowDb, - "io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph, - "org.lz4" % "lz4-java" % Versions.lz4 + "io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph ) diff --git a/commons/src/main/scala/com/github/plume/oss/util/HashUtil.scala b/commons/src/main/scala/com/github/plume/oss/util/HashUtil.scala deleted file mode 100644 index 24ccd13a..00000000 --- a/commons/src/main/scala/com/github/plume/oss/util/HashUtil.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.github.plume.oss.util - -import better.files.File -import net.jpountz.xxhash.{StreamingXXHash32, XXHashFactory} - -import java.io.InputStream -import scala.util.Using - -/** Provides methods to calculate the [[https://cyan4973.github.io/xxHash/ xxHash]] from given arguments. - */ -object HashUtil { - - private val factory: XXHashFactory = XXHashFactory.fastestInstance() - - /** Will ingest a file's contents and return the xxHash32 representation. - * - * @param f - * The file to hash. - * @return - * The given file's xxHash32 representation - */ - def getFileHash(f: File): String = getHashFromInputStream(f.newInputStream).toString - - /** Will ingest an input stream and return the xxHash32 representation. - * - * @param stream - * The stream to hash. - * @return - * The given stream's xxHash32 representation - */ - private def getHashFromInputStream(stream: InputStream): Int = { - Using.resource(stream) { inStream => - val seed = -0x68b84d74 - val hash32: StreamingXXHash32 = factory.newStreamingHash32(seed) - val buf = new Array[Byte](8192) - Iterator - .continually(inStream.read(buf)) - .takeWhile(_ != -1) - .foreach(hash32.update(buf, 0, _)) - hash32.getValue - } - } - -} diff --git a/drivers/base/src/main/scala/com/github/plume/oss/drivers/IDriver.scala b/drivers/base/src/main/scala/com/github/plume/oss/drivers/IDriver.scala index d3f5a5be..84003d9e 100644 --- a/drivers/base/src/main/scala/com/github/plume/oss/drivers/IDriver.scala +++ b/drivers/base/src/main/scala/com/github/plume/oss/drivers/IDriver.scala @@ -15,7 +15,6 @@ import scala.collection.mutable.ListBuffer */ trait IDriver extends AutoCloseable { - private val logger = LoggerFactory.getLogger(IDriver.getClass) // ID Tracking protected val currId = new AtomicLong(1) private val nodeId = TrieMap.empty[overflowdb.NodeOrDetachedNode, Long] @@ -41,10 +40,6 @@ trait IDriver extends AutoCloseable { */ def bulkTx(dg: DiffOrBuilder): Int - /** Given filenames, will remove related TYPE, TYPE_DECL, METHOD (with AST children), and NAMESPACE_BLOCK. - */ - def removeSourceFiles(filenames: String*): Unit - /** Obtains properties from the specified node type and key(s). By default will return the ID property as one of the * keys as "id". */ @@ -148,38 +143,3 @@ trait ISchemaSafeDriver extends IDriver { def buildSchemaPayload(): String } - -object IDriver { - val STRING_DEFAULT: String = "" - val INT_DEFAULT: Int = -1 - val LONG_DEFAULT: Long = -1L - val BOOL_DEFAULT: Boolean = false - val LIST_DEFAULT: Seq[String] = Seq.empty[String] - - /** Given a property, returns its known default. - */ - def getPropertyDefault(prop: String): Any = { - import PropertyNames.* - prop match { - case AST_PARENT_TYPE => STRING_DEFAULT - case AST_PARENT_FULL_NAME => STRING_DEFAULT - case NAME => STRING_DEFAULT - case CODE => STRING_DEFAULT - case ORDER => INT_DEFAULT - case SIGNATURE => "" - case ARGUMENT_INDEX => INT_DEFAULT - case FULL_NAME => STRING_DEFAULT - case TYPE_FULL_NAME => STRING_DEFAULT - case TYPE_DECL_FULL_NAME => STRING_DEFAULT - case IS_EXTERNAL => BOOL_DEFAULT - case DISPATCH_TYPE => STRING_DEFAULT - case LINE_NUMBER => INT_DEFAULT - case COLUMN_NUMBER => INT_DEFAULT - case LINE_NUMBER_END => INT_DEFAULT - case COLUMN_NUMBER_END => INT_DEFAULT - case OVERLAYS => LIST_DEFAULT - case INHERITS_FROM_TYPE_FULL_NAME => LIST_DEFAULT - case _ => STRING_DEFAULT - } - } -} diff --git a/drivers/base/src/main/scala/com/github/plume/oss/drivers/SchemaBuilder.scala b/drivers/base/src/main/scala/com/github/plume/oss/drivers/SchemaBuilder.scala new file mode 100644 index 00000000..78533437 --- /dev/null +++ b/drivers/base/src/main/scala/com/github/plume/oss/drivers/SchemaBuilder.scala @@ -0,0 +1,166 @@ +package com.github.plume.oss.drivers + +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.nodes.* +import org.slf4j.LoggerFactory + +/** A single utility to build out the CPG schema in other databases. + */ +object SchemaBuilder { + + private val logger = LoggerFactory.getLogger(getClass) + + val STRING_DEFAULT: String = "" + val INT_DEFAULT: Int = -1 + val LONG_DEFAULT: Long = -1L + val BOOL_DEFAULT: Boolean = false + val LIST_DEFAULT: Seq[String] = Seq.empty[String] + + /** Given a property, returns its known default. + */ + def getPropertyDefault(prop: String): Any = { + import io.shiftleft.codepropertygraph.generated.PropertyNames.* + prop match { + case AST_PARENT_TYPE => STRING_DEFAULT + case AST_PARENT_FULL_NAME => STRING_DEFAULT + case NAME => STRING_DEFAULT + case CODE => STRING_DEFAULT + case ORDER => INT_DEFAULT + case SIGNATURE => "" + case ARGUMENT_INDEX => INT_DEFAULT + case FULL_NAME => STRING_DEFAULT + case TYPE_FULL_NAME => STRING_DEFAULT + case TYPE_DECL_FULL_NAME => STRING_DEFAULT + case IS_EXTERNAL => BOOL_DEFAULT + case DISPATCH_TYPE => STRING_DEFAULT + case LINE_NUMBER => INT_DEFAULT + case COLUMN_NUMBER => INT_DEFAULT + case LINE_NUMBER_END => INT_DEFAULT + case COLUMN_NUMBER_END => INT_DEFAULT + case OVERLAYS => LIST_DEFAULT + case INHERITS_FROM_TYPE_FULL_NAME => LIST_DEFAULT + case POSSIBLE_TYPES => LIST_DEFAULT + case _ => STRING_DEFAULT + } + } + + /** Edges that should be specified as being between any kind of vertex. + */ + val WILDCARD_EDGE_LABELS: Set[String] = + Set(EdgeTypes.EVAL_TYPE, EdgeTypes.REF, EdgeTypes.INHERITS_FROM, EdgeTypes.ALIAS_OF) + + /** Determines if an edge type between two node types is valid. + */ + def checkEdgeConstraint(from: String, to: String, edge: String): Boolean = { + val fromCheck = from match { + case MetaData.Label => MetaData.Edges.Out.contains(edge) + case File.Label => File.Edges.Out.contains(edge) + case Method.Label => Method.Edges.Out.contains(edge) + case MethodParameterIn.Label => MethodParameterIn.Edges.Out.contains(edge) + case MethodParameterOut.Label => MethodParameterOut.Edges.Out.contains(edge) + case MethodReturn.Label => MethodReturn.Edges.Out.contains(edge) + case Modifier.Label => Modifier.Edges.Out.contains(edge) + case Type.Label => Type.Edges.Out.contains(edge) + case TypeDecl.Label => TypeDecl.Edges.Out.contains(edge) + case TypeParameter.Label => TypeParameter.Edges.Out.contains(edge) + case TypeArgument.Label => TypeArgument.Edges.Out.contains(edge) + case Member.Label => Member.Edges.Out.contains(edge) + case Namespace.Label => Namespace.Edges.Out.contains(edge) + case NamespaceBlock.Label => NamespaceBlock.Edges.Out.contains(edge) + case Literal.Label => Literal.Edges.Out.contains(edge) + case Call.Label => Call.Edges.Out.contains(edge) + case ClosureBinding.Label => ClosureBinding.Edges.Out.contains(edge) + case Local.Label => Local.Edges.Out.contains(edge) + case Identifier.Label => Identifier.Edges.Out.contains(edge) + case FieldIdentifier.Label => FieldIdentifier.Edges.Out.contains(edge) + case Return.Label => Return.Edges.Out.contains(edge) + case Block.Label => Block.Edges.Out.contains(edge) + case MethodRef.Label => MethodRef.Edges.Out.contains(edge) + case TypeRef.Label => TypeRef.Edges.Out.contains(edge) + case JumpTarget.Label => JumpTarget.Edges.Out.contains(edge) + case ControlStructure.Label => ControlStructure.Edges.Out.contains(edge) + case Annotation.Label => Annotation.Edges.Out.contains(edge) + case AnnotationLiteral.Label => AnnotationLiteral.Edges.Out.contains(edge) + case AnnotationParameter.Label => AnnotationParameter.Edges.Out.contains(edge) + case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.Out.contains(edge) + case Unknown.Label => Unknown.Edges.Out.contains(edge) + case x => + logger.warn(s"Unhandled node type '$x'") + false + } + val toCheck = to match { + case MetaData.Label => MetaData.Edges.In.contains(edge) + case File.Label => File.Edges.In.contains(edge) + case Method.Label => Method.Edges.In.contains(edge) + case MethodParameterIn.Label => MethodParameterIn.Edges.In.contains(edge) + case MethodParameterOut.Label => MethodParameterOut.Edges.In.contains(edge) + case MethodReturn.Label => MethodReturn.Edges.In.contains(edge) + case Modifier.Label => Modifier.Edges.In.contains(edge) + case Type.Label => Type.Edges.In.contains(edge) + case TypeDecl.Label => TypeDecl.Edges.In.contains(edge) + case TypeParameter.Label => TypeParameter.Edges.In.contains(edge) + case TypeArgument.Label => TypeArgument.Edges.In.contains(edge) + case Member.Label => Member.Edges.In.contains(edge) + case Namespace.Label => Namespace.Edges.In.contains(edge) + case NamespaceBlock.Label => NamespaceBlock.Edges.In.contains(edge) + case Literal.Label => Literal.Edges.In.contains(edge) + case Call.Label => Call.Edges.In.contains(edge) + case ClosureBinding.Label => ClosureBinding.Edges.Out.contains(edge) + case Local.Label => Local.Edges.In.contains(edge) + case Identifier.Label => Identifier.Edges.In.contains(edge) + case FieldIdentifier.Label => FieldIdentifier.Edges.In.contains(edge) + case Return.Label => Return.Edges.In.contains(edge) + case Block.Label => Block.Edges.In.contains(edge) + case MethodRef.Label => MethodRef.Edges.In.contains(edge) + case TypeRef.Label => TypeRef.Edges.In.contains(edge) + case JumpTarget.Label => JumpTarget.Edges.In.contains(edge) + case ControlStructure.Label => ControlStructure.Edges.In.contains(edge) + case Annotation.Label => Annotation.Edges.In.contains(edge) + case AnnotationLiteral.Label => AnnotationLiteral.Edges.In.contains(edge) + case AnnotationParameter.Label => AnnotationParameter.Edges.In.contains(edge) + case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.In.contains(edge) + case Unknown.Label => Unknown.Edges.In.contains(edge) + case x => + logger.warn(s"Unhandled node type '$x'") + false + } + + fromCheck && toCheck + } + + def allProperties: Set[String] = NodeToProperties.flatMap(_._2).toSet + + val NodeToProperties: Map[String, Set[String]] = Map( + MetaData.Label -> MetaData.PropertyNames.all, + File.Label -> File.PropertyNames.all, + Method.Label -> Method.PropertyNames.all, + MethodParameterIn.Label -> MethodParameterIn.PropertyNames.all, + MethodParameterOut.Label -> MethodParameterOut.PropertyNames.all, + MethodReturn.Label -> MethodReturn.PropertyNames.all, + Modifier.Label -> Modifier.PropertyNames.all, + Type.Label -> Type.PropertyNames.all, + TypeDecl.Label -> TypeDecl.PropertyNames.all, + TypeParameter.Label -> TypeParameter.PropertyNames.all, + TypeArgument.Label -> TypeArgument.PropertyNames.all, + Member.Label -> Member.PropertyNames.all, + Namespace.Label -> Namespace.PropertyNames.all, + NamespaceBlock.Label -> NamespaceBlock.PropertyNames.all, + Literal.Label -> Literal.PropertyNames.all, + Call.Label -> Call.PropertyNames.all, + Local.Label -> Local.PropertyNames.all, + Identifier.Label -> Identifier.PropertyNames.all, + FieldIdentifier.Label -> FieldIdentifier.PropertyNames.all, + Return.Label -> Return.PropertyNames.all, + Block.Label -> Block.PropertyNames.all, + MethodRef.Label -> MethodRef.PropertyNames.all, + TypeRef.Label -> TypeRef.PropertyNames.all, + JumpTarget.Label -> JumpTarget.PropertyNames.all, + ControlStructure.Label -> ControlStructure.PropertyNames.all, + Annotation.Label -> Annotation.PropertyNames.all, + AnnotationLiteral.Label -> AnnotationLiteral.PropertyNames.all, + AnnotationParameter.Label -> AnnotationParameter.PropertyNames.all, + AnnotationParameterAssign.Label -> AnnotationParameterAssign.PropertyNames.all, + Unknown.Label -> Unknown.PropertyNames.all + ) + +} diff --git a/drivers/base/src/test/scala/com/github/plume/oss/DockerManager.scala b/drivers/base/src/test/scala/com/github/plume/oss/DockerManager.scala index 74c8a22c..14cdae6a 100644 --- a/drivers/base/src/test/scala/com/github/plume/oss/DockerManager.scala +++ b/drivers/base/src/test/scala/com/github/plume/oss/DockerManager.scala @@ -3,7 +3,7 @@ package com.github.plume.oss import io.circe.Json import org.slf4j.LoggerFactory -import java.io.{File => JavaFile} +import better.files.File import scala.collection.mutable.ListBuffer import scala.sys.process.{Process, ProcessLogger, stringSeqToProcess} @@ -11,23 +11,25 @@ object DockerManager { private val logger = LoggerFactory.getLogger(getClass) - def toDockerComposeFile(dbName: String): JavaFile = - new JavaFile(getClass.getResource(s"/docker/$dbName.yml").toURI) + def toDockerComposeFile(dbName: String)(implicit classLoader: ClassLoader): File = + File(classLoader.getResource(s"docker/$dbName.yml").toURI) - def closeAnyDockerContainers(dbName: String): Unit = { + def closeAnyDockerContainers(dbName: String)(implicit classLoader: ClassLoader): Unit = { logger.info(s"Stopping Docker services for $dbName...") - val dockerComposeUp = Process(Seq("docker-compose", "-f", toDockerComposeFile(dbName).getAbsolutePath, "down")) - dockerComposeUp.run(ProcessLogger(_ => ())) + val dockerComposeDown = Process(Seq("docker", "compose", "-f", toDockerComposeFile(dbName).pathAsString, "down")) + dockerComposeDown.run(ProcessLogger(_ => ())) } - def startDockerFile(dbName: String, containers: List[String] = List.empty[String]): Unit = { + def startDockerFile(dbName: String, containers: List[String] = List.empty[String])(implicit + classLoader: ClassLoader + ): Unit = { val healthChecks = ListBuffer.empty[String] if (containers.isEmpty) healthChecks += dbName else healthChecks ++= containers logger.info(s"Docker Compose file found for $dbName, starting...") closeAnyDockerContainers(dbName) // Easiest way to clear the db Thread.sleep(3000) val dockerComposeUp = Process( - Seq("docker-compose", "-f", toDockerComposeFile(dbName).getAbsolutePath, "up", "--remove-orphans") + Seq("docker", "compose", "-f", toDockerComposeFile(dbName).pathAsString, "up", "--remove-orphans") ) logger.info(s"Starting process $dockerComposeUp") dockerComposeUp.run(ProcessLogger(_ => ())) diff --git a/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala b/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala index af6eeb40..9aa0c0eb 100644 --- a/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala +++ b/drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala @@ -34,7 +34,7 @@ class PlumeDriverFixture(val driver: IDriver) val props: Array[Object] = n.properties.flatMap { case (k, v) => Iterable(k.asInstanceOf[Object], v.asInstanceOf[Object]) }.toArray - new DetachedNodeGeneric(n.label(), props: _*) + new DetachedNodeGeneric(n.label(), props*) } "overflowdb.BatchedUpdate.DiffGraph based changes" should { @@ -97,23 +97,6 @@ class PlumeDriverFixture(val driver: IDriver) } } - "should delete a source file's nodes precisely" in { - val diffGraph = new DiffGraphBuilder - // Create some basic method - createSimpleGraph(diffGraph) - driver.bulkTx(diffGraph.build()) - - // remove f1 nodes - driver.removeSourceFiles(f1.name) - - val List(m: Map[String, Any]) = driver.propertyFromNodes(METHOD, NAME) - val List(td: Map[String, Any]) = driver.propertyFromNodes(TYPE_DECL, NAME) - val List(n: Map[String, Any]) = driver.propertyFromNodes(NAMESPACE_BLOCK, NAME) - m.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe m2.name - td.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe td2.name - n.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe n2.name - } - override def afterAll(): Unit = { if (driver.isConnected) driver.close() } diff --git a/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala b/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala index 96c5f316..7681995d 100644 --- a/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala +++ b/drivers/gremlin/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala @@ -133,50 +133,11 @@ abstract class GremlinDriver(txMax: Int = 50) extends IDriver { } } - override def removeSourceFiles(filenames: String*): Unit = { - val fs = g() - .V() - .hasLabel(NodeTypes.FILE) - .filter(__.has(PropertyNames.NAME, within[String](filenames: _*))) - .id() - .toSet - .asScala - .toSeq - - g() - .V(fs: _*) - .in(EdgeTypes.SOURCE_FILE) - .filter(__.hasLabel(NodeTypes.TYPE_DECL)) - .in(EdgeTypes.REF) - .drop() - .iterate() - - g() - .V(fs: _*) - .in(EdgeTypes.SOURCE_FILE) - .hasLabel(NodeTypes.NAMESPACE_BLOCK) - .aggregate("x") - .repeat(__.out(EdgeTypes.AST, EdgeTypes.CONDITION)) - .emit() - .barrier() - .aggregate("x") - .select[Vertex]("x") - .unfold[Vertex]() - .dedup() - .drop() - .iterate() - - g() - .V(fs: _*) - .drop() - .iterate() - } - override def propertyFromNodes(nodeType: String, keys: String*): List[Map[String, Any]] = { var ptr = g() .V() .hasLabel(nodeType) - .project[Any](T.id.toString, keys: _*) + .project[Any](T.id.toString, keys*) .by(T.id) keys.foreach(k => ptr = ptr.by(coalesce(values(k), constant("NULL")))) ptr.asScala @@ -184,7 +145,7 @@ abstract class GremlinDriver(txMax: Int = 50) extends IDriver { _.asScala .map { case (k, v) => if (v == "NULL") - k -> IDriver.getPropertyDefault(k) + k -> SchemaBuilder.getPropertyDefault(k) else if (v == PropertyNames.OVERLAYS || v == PropertyNames.INHERITS_FROM_TYPE_FULL_NAME) k -> v.toString.split(",").toSeq else diff --git a/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala b/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala index 72abfe11..8c990261 100644 --- a/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala +++ b/drivers/neo4j/src/main/scala/com/github/plume/oss/drivers/Neo4jDriver.scala @@ -148,7 +148,7 @@ final class Neo4jDriver( .map { case (key: String, value: Any, node: StoredNode) => val newV = value match { case x: String => "\"" + x + "\"" - case Seq() => IDriver.STRING_DEFAULT + case Seq() => SchemaBuilder.STRING_DEFAULT case xs: Seq[_] => "[" + xs.map { x => Seq("\"", x, "\"").mkString }.mkString(",") + "]" case x: Number => x.toString @@ -223,46 +223,6 @@ final class Neo4jDriver( dg.size() } - /** Removes the namespace block with all of its AST children specified by the given FILENAME property. - */ - private def deleteNamespaceBlockWithAstChildrenByFilename(filename: String): Unit = - Using.resource(driver.session()) { session => - session.writeTransaction { tx => - tx - .run( - s""" - |MATCH (a:${NodeTypes.NAMESPACE_BLOCK})-[r:${EdgeTypes.AST}*]->(t) - |WHERE a.${PropertyNames.FILENAME} = $$filename - |FOREACH (x IN r | DELETE x) - |DETACH DELETE a, t - |""".stripMargin, - new util.HashMap[String, Object](1) { put("filename", filename.asInstanceOf[Object]) } - ) - } - } - - override def removeSourceFiles(filenames: String*): Unit = { - Using.resource(driver.session()) { session => - val fileSet = CollectionConverters.IterableHasAsJava(filenames.toSeq).asJava - session.writeTransaction { tx => - val filePayload = s""" - |MATCH (f:${NodeTypes.FILE}) - |MATCH (f)<-[:${EdgeTypes.SOURCE_FILE}]-(td:${NodeTypes.TYPE_DECL})<-[:${EdgeTypes.REF}]-(t) - |WHERE f.NAME IN $$fileSet - |DETACH DELETE f, t - |""".stripMargin - runPayload( - tx, - filePayload, - new util.HashMap[String, Object](1) { - put("fileSet", fileSet) - } - ) - } - } - filenames.foreach(deleteNamespaceBlockWithAstChildrenByFilename) - } - private def runPayload( tx: Transaction, filePayload: String, @@ -289,15 +249,15 @@ final class Neo4jDriver( (keys :+ "id").flatMap { k => val v = record.get(k) if (v.hasType(typeSystem.NULL())) { - Some(k -> IDriver.getPropertyDefault(k)) + Some(k -> SchemaBuilder.getPropertyDefault(k)) } else if (k == "id") { - Some(k -> v.asLong(IDriver.LONG_DEFAULT)) + Some(k -> v.asLong(SchemaBuilder.LONG_DEFAULT)) } else if (v.hasType(typeSystem.INTEGER())) { - Some(k -> v.asInt(IDriver.INT_DEFAULT)) + Some(k -> v.asInt(SchemaBuilder.INT_DEFAULT)) } else if (v.hasType(typeSystem.BOOLEAN())) { - Some(k -> v.asBoolean(IDriver.BOOL_DEFAULT)) + Some(k -> v.asBoolean(SchemaBuilder.BOOL_DEFAULT)) } else if (v.hasType(typeSystem.STRING())) { - Some(k -> v.asString(IDriver.STRING_DEFAULT)) + Some(k -> v.asString(SchemaBuilder.STRING_DEFAULT)) } else if (v.hasType(typeSystem.LIST())) { Some(k -> v.asList()) } else { diff --git a/drivers/neo4j/src/test/resources/docker/Neo4j.yml b/drivers/neo4j/src/test/resources/docker/Neo4j.yml index 41ed5d6b..a67114d2 100644 --- a/drivers/neo4j/src/test/resources/docker/Neo4j.yml +++ b/drivers/neo4j/src/test/resources/docker/Neo4j.yml @@ -1,4 +1,4 @@ -version: '3' + services: neo4j: image: bitnami/neo4j:4.3.6 diff --git a/drivers/neo4j/src/test/scala/com/github/plume/oss/drivers/Neo4jIntTests.scala b/drivers/neo4j/src/test/scala/com/github/plume/oss/drivers/Neo4jIntTests.scala index 53a86787..703b421d 100644 --- a/drivers/neo4j/src/test/scala/com/github/plume/oss/drivers/Neo4jIntTests.scala +++ b/drivers/neo4j/src/test/scala/com/github/plume/oss/drivers/Neo4jIntTests.scala @@ -5,9 +5,11 @@ import com.github.plume.oss.testfixtures.PlumeDriverFixture class Neo4jIntTests extends PlumeDriverFixture(new Neo4jDriver()) { + private implicit val loader: ClassLoader = getClass.getClassLoader + override def beforeAll(): Unit = { DockerManager.startDockerFile("Neo4j", List("plume-neo4j")) - driver.asInstanceOf[IDriver with ISchemaSafeDriver].buildSchema() + driver.asInstanceOf[IDriver & ISchemaSafeDriver].buildSchema() } override def afterAll(): Unit = { diff --git a/drivers/overflowdb/build.sbt b/drivers/overflowdb/build.sbt index 436cd6be..c3731a7d 100644 --- a/drivers/overflowdb/build.sbt +++ b/drivers/overflowdb/build.sbt @@ -3,10 +3,9 @@ name := "overflowdb" dependsOn(Projects.base, Projects.base % "compile->compile;test->test", Projects.commons) libraryDependencies ++= Seq( - "io.shiftleft" %% "overflowdb-traversal" % Versions.overflowDb, - "io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph, - "io.joern" %% "semanticcpg" % Versions.joern, - "io.joern" %% "dataflowengineoss" % Versions.joern, + "io.shiftleft" %% "overflowdb-traversal" % Versions.overflowDb, + "io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph, + "org.apache.commons" % "commons-text" % Versions.apacheCommonsText, "org.apache.tinkerpop" % "gremlin-driver" % Versions.tinkerGraph % Test, "org.apache.tinkerpop" % "tinkergraph-gremlin" % Versions.tinkerGraph % Test ) diff --git a/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala b/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala index ad07692a..c950c483 100644 --- a/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala +++ b/drivers/overflowdb/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala @@ -6,7 +6,7 @@ import com.github.plume.oss.util.BatchedUpdateUtil import com.github.plume.oss.util.BatchedUpdateUtil.* import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import org.apache.commons.lang.StringEscapeUtils +import org.apache.commons.text.StringEscapeUtils import org.slf4j.LoggerFactory import overflowdb.BatchedUpdate.DiffOrBuilder import overflowdb.traversal.jIteratortoTraversal @@ -93,36 +93,6 @@ final case class OverflowDbDriver( dg.size() } - private def accumNodesToDelete(n: Node, visitedNodes: mutable.Set[Node], edgeToFollow: String*): Unit = { - if (!visitedNodes.contains(n)) { - visitedNodes.add(n) - n.out(edgeToFollow: _*) - .forEachRemaining(accumNodesToDelete(_, visitedNodes, edgeToFollow: _*)) - } - } - - override def removeSourceFiles(filenames: String*): Unit = { - val fs = filenames.toSet - cpg.graph - .nodes(NodeTypes.FILE) - .filter { f => - fs.contains(f.property(PropertyNames.NAME).toString) - } - .foreach { f => - val fileChildren = f.in(EdgeTypes.SOURCE_FILE).asScala.toList - val typeDecls = fileChildren.collect { case x: TypeDecl => x } - val namespaceBlocks = fileChildren.collect { case x: NamespaceBlock => x } - // Remove TYPE nodes - typeDecls.flatMap(_.in(EdgeTypes.REF)).foreach(safeRemove) - // Remove NAMESPACE_BLOCKs and their AST children (TYPE_DECL, METHOD, etc.) - val nodesToDelete = mutable.Set.empty[Node] - namespaceBlocks.foreach(accumNodesToDelete(_, nodesToDelete, EdgeTypes.AST, EdgeTypes.CONDITION)) - nodesToDelete.foreach(safeRemove) - // Finally remove FILE node - safeRemove(f) - } - } - private def safeRemove(n: Node): Unit = Try(if (n != null) { n.inE().forEachRemaining(_.remove()) n.outE().forEachRemaining(_.remove()) @@ -191,7 +161,7 @@ final case class OverflowDbDriver( osw.write("") osw.write("" + n.label() + "") serializeLists(n.propertiesMap().asScala.toMap).foreach { case (k, v) => - osw.write("" + StringEscapeUtils.escapeXml(v.toString) + "") + osw.write("" + StringEscapeUtils.escapeXml11(v.toString) + "") } osw.write("") } diff --git a/drivers/tigergraph/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala b/drivers/tigergraph/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala index 3cfd8a93..07d9461b 100644 --- a/drivers/tigergraph/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala +++ b/drivers/tigergraph/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala @@ -32,7 +32,7 @@ final class TigerGraphDriver( timeout: Int = DEFAULT_TIMEOUT, scheme: String = "http", txMax: Int = DEFAULT_TX_MAX, - tgVersion: String = "3.5.0", + tgVersion: String = "3.10.1", authKey: String = "" ) extends IDriver with ISchemaSafeDriver { @@ -117,7 +117,7 @@ final class TigerGraphDriver( val attributes = properties.flatMap { case (k, v) => val vStr = v match { case xs: Seq[_] => xs.mkString(",") - case x if x == null => IDriver.getPropertyDefault(k) + case x if x == null => SchemaBuilder.getPropertyDefault(k) case x => x } jsonValue(vStr) match { @@ -210,14 +210,11 @@ final class TigerGraphDriver( post("graph/cpg", PayloadBody(edges = payload)) } - override def removeSourceFiles(filenames: String*): Unit = - get("query/cpg/delete_source_file", filenames.map(("filenames", _))) - override def propertyFromNodes(nodeType: String, keys: String*): List[Map[String, Any]] = { keys .map { k => val params = Map("node_type" -> s"${nodeType}_", "property" -> s"_$k") - IDriver.getPropertyDefault(k) match { + SchemaBuilder.getPropertyDefault(k) match { case _: Boolean => k -> get("query/cpg/b_property_from_nodes", params) case _: Int => k -> get("query/cpg/i_property_from_nodes", params) case _ => k -> get("query/cpg/s_property_from_nodes", params) @@ -243,14 +240,14 @@ final class TigerGraphDriver( override def buildSchemaPayload(): String = { s""" - |DROP ALL - |$VERTICES - |$EDGES - |$WILDCARD_EDGES - |CREATE GRAPH cpg(*) - |$QUERIES - |INSTALL QUERY ALL - |""".stripMargin + |DROP ALL + |$VERTICES + |$EDGES + |$WILDCARD_EDGES + |CREATE GRAPH cpg(*) + |$QUERIES + |INSTALL QUERY ALL + |""".stripMargin } private def request(): RequestT[Empty, Either[String, String], Any] = @@ -298,7 +295,7 @@ final class TigerGraphDriver( } private def get(endpoint: String, params: Seq[(String, String)]): Seq[Json] = { - val uri = buildUri(endpoint).addParams(params: _*) + val uri = buildUri(endpoint).addParams(params*) Try( request() .get(uri) @@ -345,17 +342,19 @@ final class TigerGraphDriver( } private def postGSQL(payload: String): Unit = { - Try(sys.env("GSQL_HOME")) match { + Try(sys.env("GSQL_HOME")).map(x => better.files.File(x.trim)) match { case Failure(_) => throw new RuntimeException(""" - |Environment variable 'GSQL_HOME' not found on the OS. Please install the gsql_client.jar - |from https://docs.tigergraph.com/tigergraph-server/current/gsql-shell/using-a-remote-gsql-client and set the - |path to the JAR file as GSQL_HOME. - |""".stripMargin) - case Success(gsqlPath) => + |Environment variable 'GSQL_HOME' not found on the OS. Please install the gsql_client.jar + |from https://docs.tigergraph.com/tigergraph-server/current/gsql-shell/using-a-remote-gsql-client and set the + |path to the JAR file as GSQL_HOME. + |""".stripMargin) + case Success(gsqlJar) if !gsqlJar.exists => + throw new RuntimeException(s"Path defined by `GSQL_HOME` (${gsqlJar.pathAsString}) does not exist!") + case Success(gsqlJar) => val args = Seq("-ip", s"$hostname:$gsqlPort", "-u", username, "-p", password, payload) logger.debug(s"Posting payload:\n$payload") - executeGsqlClient(gsqlPath, args) + executeGsqlClient(gsqlJar.pathAsString, args) } } @@ -365,8 +364,12 @@ final class TigerGraphDriver( val allLogs = new StringBuilder() val processLogger = ProcessLogger.apply( - (s: String) => { logger.info(s); allLogs.append(s) }, - (s: String) => { logger.error(s); allLogs.append(s) } + (s: String) => { + logger.info(s); allLogs.append(s) + }, + (s: String) => { + logger.error(s); allLogs.append(s) + } ) Try(sys.env("JAVA_HOME")) match { case Failure(_) => @@ -427,7 +430,7 @@ object TigerGraphDriver { /** Returns the corresponding TigerGraph type given a Scala type. */ private def odbToTgType(propKey: String): String = { - IDriver.getPropertyDefault(propKey) match { + SchemaBuilder.getPropertyDefault(propKey) match { case _: Long => "UINT" case _: Int => "INT" case _: Boolean => "BOOL" @@ -435,84 +438,6 @@ object TigerGraphDriver { } } - /** Edges that should be specified as being between any kind of vertex. - */ - private val WILDCARD_EDGE_LABELS = - Set(EdgeTypes.EVAL_TYPE, EdgeTypes.REF, EdgeTypes.INHERITS_FROM, EdgeTypes.ALIAS_OF) - - /** Determines if an edge type between two node types is valid. - */ - def checkEdgeConstraint(from: String, to: String, edge: String): Boolean = { - val fromCheck = from match { - case MetaData.Label => MetaData.Edges.Out.contains(edge) - case File.Label => File.Edges.Out.contains(edge) - case Method.Label => Method.Edges.Out.contains(edge) - case MethodParameterIn.Label => MethodParameterIn.Edges.Out.contains(edge) - case MethodParameterOut.Label => MethodParameterOut.Edges.Out.contains(edge) - case MethodReturn.Label => MethodReturn.Edges.Out.contains(edge) - case Modifier.Label => Modifier.Edges.Out.contains(edge) - case Type.Label => Type.Edges.Out.contains(edge) - case TypeDecl.Label => TypeDecl.Edges.Out.contains(edge) - case TypeParameter.Label => TypeParameter.Edges.Out.contains(edge) - case TypeArgument.Label => TypeArgument.Edges.Out.contains(edge) - case Member.Label => Member.Edges.Out.contains(edge) - case Namespace.Label => Namespace.Edges.Out.contains(edge) - case NamespaceBlock.Label => NamespaceBlock.Edges.Out.contains(edge) - case Literal.Label => Literal.Edges.Out.contains(edge) - case Call.Label => Call.Edges.Out.contains(edge) - case Local.Label => Local.Edges.Out.contains(edge) - case Identifier.Label => Identifier.Edges.Out.contains(edge) - case FieldIdentifier.Label => FieldIdentifier.Edges.Out.contains(edge) - case Return.Label => Return.Edges.Out.contains(edge) - case Block.Label => Block.Edges.Out.contains(edge) - case MethodRef.Label => MethodRef.Edges.Out.contains(edge) - case TypeRef.Label => TypeRef.Edges.Out.contains(edge) - case JumpTarget.Label => JumpTarget.Edges.Out.contains(edge) - case ControlStructure.Label => ControlStructure.Edges.Out.contains(edge) - case Annotation.Label => Annotation.Edges.Out.contains(edge) - case AnnotationLiteral.Label => AnnotationLiteral.Edges.Out.contains(edge) - case AnnotationParameter.Label => AnnotationParameter.Edges.Out.contains(edge) - case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.Out.contains(edge) - case Unknown.Label => Unknown.Edges.Out.contains(edge) - case _ => false - } - val toCheck = to match { - case MetaData.Label => MetaData.Edges.In.contains(edge) - case File.Label => File.Edges.In.contains(edge) - case Method.Label => Method.Edges.In.contains(edge) - case MethodParameterIn.Label => MethodParameterIn.Edges.In.contains(edge) - case MethodParameterOut.Label => MethodParameterOut.Edges.In.contains(edge) - case MethodReturn.Label => MethodReturn.Edges.In.contains(edge) - case Modifier.Label => Modifier.Edges.In.contains(edge) - case Type.Label => Type.Edges.In.contains(edge) - case TypeDecl.Label => TypeDecl.Edges.In.contains(edge) - case TypeParameter.Label => TypeParameter.Edges.In.contains(edge) - case TypeArgument.Label => TypeArgument.Edges.In.contains(edge) - case Member.Label => Member.Edges.In.contains(edge) - case Namespace.Label => Namespace.Edges.In.contains(edge) - case NamespaceBlock.Label => NamespaceBlock.Edges.In.contains(edge) - case Literal.Label => Literal.Edges.In.contains(edge) - case Call.Label => Call.Edges.In.contains(edge) - case Local.Label => Local.Edges.In.contains(edge) - case Identifier.Label => Identifier.Edges.In.contains(edge) - case FieldIdentifier.Label => FieldIdentifier.Edges.In.contains(edge) - case Return.Label => Return.Edges.In.contains(edge) - case Block.Label => Block.Edges.In.contains(edge) - case MethodRef.Label => MethodRef.Edges.In.contains(edge) - case TypeRef.Label => TypeRef.Edges.In.contains(edge) - case JumpTarget.Label => JumpTarget.Edges.In.contains(edge) - case ControlStructure.Label => ControlStructure.Edges.In.contains(edge) - case Annotation.Label => Annotation.Edges.In.contains(edge) - case AnnotationLiteral.Label => AnnotationLiteral.Edges.In.contains(edge) - case AnnotationParameter.Label => AnnotationParameter.Edges.In.contains(edge) - case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.In.contains(edge) - case Unknown.Label => Unknown.Edges.In.contains(edge) - case _ => false - } - - fromCheck && toCheck - } - /** Edges as a schema string. Each edge is prepended with "_" to escape reserved words. */ private def EDGES: String = { @@ -520,13 +445,13 @@ object TigerGraphDriver { .flatMap { e => NodeTypes.ALL.asScala.flatMap { src => NodeTypes.ALL.asScala.flatMap { dst => - if (checkEdgeConstraint(src, dst, e)) Some((src, dst, e)) + if (SchemaBuilder.checkEdgeConstraint(src, dst, e)) Some((src, dst, e)) else None } } } .groupBy { case (_, _, e) => e } - .filterNot { x => WILDCARD_EDGE_LABELS.contains(x._1) } + .filterNot { x => SchemaBuilder.WILDCARD_EDGE_LABELS.contains(x._1) } .map { case (label, xs) => val prefix = s"CREATE DIRECTED EDGE _$label(" val body = xs.map { case (src, dst, _) => s"FROM ${src}_, TO ${dst}_" }.mkString("|") @@ -538,10 +463,8 @@ object TigerGraphDriver { /** Creates the schema string of all edges that should be treated as widlcards. */ private def WILDCARD_EDGES: String = { - WILDCARD_EDGE_LABELS - .map { x => - s"CREATE DIRECTED EDGE _$x(FROM *, TO *)" - } + SchemaBuilder.WILDCARD_EDGE_LABELS + .map(x => s"CREATE DIRECTED EDGE _$x(FROM *, TO *)") .mkString("\n") } @@ -549,7 +472,7 @@ object TigerGraphDriver { */ private def VERTICES: String = { def propToTg(x: String) = { - val default = IDriver.getPropertyDefault(x) match { + val default = SchemaBuilder.getPropertyDefault(x) match { case x: String => "\"" + x + "\"" case x: Boolean => "\"" + x + "\"" case _: Seq[_] => "\"" + "" + "\"" @@ -557,40 +480,12 @@ object TigerGraphDriver { } s"_$x ${odbToTgType(x)} DEFAULT $default" } + def vertexSchema(label: String, props: Set[String]): String = s"CREATE VERTEX ${label}_ (PRIMARY_ID id UINT, ${props.map(propToTg).mkString(",")}) WITH primary_id_as_attribute=" + "\"true\"" s""" - |${vertexSchema(MetaData.Label, MetaData.PropertyNames.all)} - |${vertexSchema(File.Label, File.PropertyNames.all)} - |${vertexSchema(Method.Label, Method.PropertyNames.all)} - |${vertexSchema(MethodParameterIn.Label, MethodParameterIn.PropertyNames.all)} - |${vertexSchema(MethodParameterOut.Label, MethodParameterOut.PropertyNames.all)} - |${vertexSchema(MethodReturn.Label, MethodReturn.PropertyNames.all)} - |${vertexSchema(Modifier.Label, Modifier.PropertyNames.all)} - |${vertexSchema(Type.Label, Type.PropertyNames.all)} - |${vertexSchema(TypeDecl.Label, TypeDecl.PropertyNames.all)} - |${vertexSchema(TypeParameter.Label, TypeParameter.PropertyNames.all)} - |${vertexSchema(TypeArgument.Label, TypeArgument.PropertyNames.all)} - |${vertexSchema(Member.Label, Member.PropertyNames.all)} - |${vertexSchema(Namespace.Label, Namespace.PropertyNames.all)} - |${vertexSchema(NamespaceBlock.Label, NamespaceBlock.PropertyNames.all)} - |${vertexSchema(Literal.Label, Literal.PropertyNames.all)} - |${vertexSchema(Call.Label, Call.PropertyNames.all)} - |${vertexSchema(Local.Label, Local.PropertyNames.all)} - |${vertexSchema(Identifier.Label, Identifier.PropertyNames.all)} - |${vertexSchema(FieldIdentifier.Label, FieldIdentifier.PropertyNames.all)} - |${vertexSchema(Return.Label, Return.PropertyNames.all)} - |${vertexSchema(Block.Label, Block.PropertyNames.all)} - |${vertexSchema(MethodRef.Label, MethodRef.PropertyNames.all)} - |${vertexSchema(TypeRef.Label, TypeRef.PropertyNames.all)} - |${vertexSchema(JumpTarget.Label, JumpTarget.PropertyNames.all)} - |${vertexSchema(ControlStructure.Label, ControlStructure.PropertyNames.all)} - |${vertexSchema(Annotation.Label, Annotation.PropertyNames.all)} - |${vertexSchema(AnnotationLiteral.Label, AnnotationLiteral.PropertyNames.all)} - |${vertexSchema(AnnotationParameter.Label, AnnotationParameter.PropertyNames.all)} - |${vertexSchema(AnnotationParameterAssign.Label, AnnotationParameterAssign.PropertyNames.all)} - |${vertexSchema(Unknown.Label, Unknown.PropertyNames.all)} - |""".stripMargin + |${SchemaBuilder.NodeToProperties.map(vertexSchema).mkString("\n")} + |""".stripMargin } private def QUERIES: String = { @@ -633,88 +528,20 @@ object TigerGraphDriver { | FROM seed:src | WHERE src.id IN ids; |} - |""".stripMargin, - """ - |CREATE QUERY id_interval(INT lower, INT upper) FOR GRAPH cpg { - | SetAccum @@ids; - | seed = {ANY}; - | temp = SELECT src - | FROM seed:src - | WHERE src.id >= lower AND src.id <= upper - | ACCUM @@ids += src.id; - | PRINT @@ids as ids; - |} - |""".stripMargin, - """ - |CREATE QUERY delete_source_file(SET filenames) FOR GRAPH cpg SYNTAX v2 { - | fs = {FILE_.*}; - | fvs = SELECT f - | FROM fs:f - | WHERE f._NAME IN filenames; - | tds = SELECT td - | FROM fvs -(<_SOURCE_FILE)- TYPE_DECL_:td; - | ts = SELECT t - | FROM tds -(<_REF)- TYPE_:t; - | nbs = SELECT n - | FROM fvs - (<_SOURCE_FILE)- NAMESPACE_BLOCK_:n; - | - | childVs = SELECT t - | FROM nbs:s -((_AST>|_CONDITION>)*) - :t; - | - | nsToDelete = fvs UNION tds UNION ts UNION childVs; - | DELETE v FROM nsToDelete:v; - |} - |""".stripMargin, - """ - |CREATE QUERY static_call_linker() FOR GRAPH cpg { - | TYPEDEF TUPLE> M_TUP; - | SetAccum @@methods; - | calls = {CALL_.*}; - | methods = {METHOD_.*}; - | - | ms = SELECT m - | FROM methods:m - | ACCUM @@methods += M_TUP(m._FULL_NAME, m); - | - | FOREACH m in @@methods DO - | cs = SELECT c - | FROM calls:c - | WHERE c._DISPATCH_TYPE == "STATIC_DISPATCH" - | AND m.FULL_NAME == c._METHOD_FULL_NAME - | ACCUM INSERT INTO _CALL VALUES(c, m.method); - | END; - |} - |""".stripMargin - ) ++ Array( - (EdgeTypes.REF, PropertyNames.NAME), - (EdgeTypes.REF, PropertyNames.FULL_NAME), - (EdgeTypes.EVAL_TYPE, PropertyNames.TYPE_FULL_NAME), - (EdgeTypes.REF, PropertyNames.METHOD_FULL_NAME), - (EdgeTypes.INHERITS_FROM, PropertyNames.INHERITS_FROM_TYPE_FULL_NAME), - (EdgeTypes.ALIAS_OF, PropertyNames.ALIAS_TYPE_FULL_NAME) - ).map { case (e: String, p: String) => - s""" - |CREATE QUERY link_ast_${e.toLowerCase}_${p.toLowerCase}(SET src_labels, STRING dst_value, VERTEX dst) FOR GRAPH cpg { - | seed = {ANY}; - | temp = SELECT src - | FROM seed:src - | WHERE src.type IN src_labels AND src._$p LIKE dst_value - | ACCUM INSERT INTO _$e VALUES (src, dst); - |} |""".stripMargin - } ++ + ) ++ Array("STRING", "INT", "BOOL").map(x => s""" - |CREATE QUERY ${x(0).toLower}_property_from_nodes(STRING node_type, STRING property) FOR GRAPH cpg { - | TYPEDEF TUPLE RES; - | SetAccum @@result; - | seed = {ANY}; - | temp = SELECT src - | FROM seed:src - | WHERE src.type == node_type - | ACCUM @@result += RES(src.id, src.getAttr(property, "$x")); - | PRINT @@result as properties; - |} - |""".stripMargin)).mkString + |CREATE QUERY ${x(0).toLower}_property_from_nodes(STRING node_type, STRING property) FOR GRAPH cpg { + | TYPEDEF TUPLE RES; + | SetAccum @@result; + | seed = {ANY}; + | temp = SELECT src + | FROM seed:src + | WHERE src.type == node_type + | ACCUM @@result += RES(src.id, src.getAttr(property, "$x")); + | PRINT @@result as properties; + |} + |""".stripMargin)).mkString } } diff --git a/drivers/tigergraph/src/test/resources/docker/Dockerfile b/drivers/tigergraph/src/test/resources/docker/Dockerfile deleted file mode 100644 index 571fa521..00000000 --- a/drivers/tigergraph/src/test/resources/docker/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM tigergraph/tigergraph:3.9.3-1 -ENTRYPOINT /usr/sbin/sshd && \ - su - tigergraph bash -c "/home/tigergraph/tigergraph/app/cmd/gadmin start all && tail -f /dev/null" \ No newline at end of file diff --git a/drivers/tigergraph/src/test/resources/docker/TigerGraph.yml b/drivers/tigergraph/src/test/resources/docker/TigerGraph.yml index b405bfd6..257ddecb 100644 --- a/drivers/tigergraph/src/test/resources/docker/TigerGraph.yml +++ b/drivers/tigergraph/src/test/resources/docker/TigerGraph.yml @@ -1,7 +1,7 @@ -version: '3' + services: tigergraph: - build: . + image: tigergraph/tigergraph:3.10.1 container_name: plume-tigergraph ports: - "14022:22" diff --git a/drivers/tigergraph/src/test/scala/com/github/plume/oss/drivers/TigerGraphIntTests.scala b/drivers/tigergraph/src/test/scala/com/github/plume/oss/drivers/TigerGraphIntTests.scala index 7d96ec8b..6ac20df7 100644 --- a/drivers/tigergraph/src/test/scala/com/github/plume/oss/drivers/TigerGraphIntTests.scala +++ b/drivers/tigergraph/src/test/scala/com/github/plume/oss/drivers/TigerGraphIntTests.scala @@ -5,13 +5,15 @@ import com.github.plume.oss.testfixtures.PlumeDriverFixture class TigerGraphIntTests extends PlumeDriverFixture(new TigerGraphDriver()) { + private implicit val loader: ClassLoader = getClass.getClassLoader + override def beforeAll(): Unit = { - DockerManager.startDockerFile("TigerGraph", List("plume-tigergraph")) - driver.asInstanceOf[IDriver with ISchemaSafeDriver].buildSchema() +// DockerManager.startDockerFile("TigerGraph", List("plume-tigergraph")) + driver.asInstanceOf[IDriver & ISchemaSafeDriver].buildSchema() } override def afterAll(): Unit = { - DockerManager.closeAnyDockerContainers("TigerGraph") +// DockerManager.closeAnyDockerContainers("TigerGraph") } } diff --git a/project/Versions.scala b/project/Versions.scala index 775774a4..2a6c624f 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -1,25 +1,27 @@ object Versions { // SAST - val codePropertyGraph = "1.4.33" - val joern = "2.0.220" - val overflowDb = "1.181" + val codePropertyGraph = "1.6.11" + val joern = "2.0.379" // Drivers val tinkerGraph = "3.4.11" - val neo4j = "4.4.5" + val neo4j = "4.4.5" + val overflowDb = "1.192" + val flatGraph = "0.0.59" // Utilities val apacheCodec = "1.15" - val apacheIo = "2.11.0" - val apacheLang = "3.12.0" - val sttp = "3.9.0" - val jackson = "2.13.2" - val lz4 = "1.8.0" - val slf4j = "2.0.5" - val log4j = "2.20.0" - val logback = "1.2.11" - val scalatest = "3.2.15" - val circe = "0.14.1" + val apacheIo = "2.11.0" + val apacheLang = "3.12.0" + val apacheCommonsText = "1.12.0" + val sttp = "3.9.0" + val jackson = "2.13.2" + val lz4 = "1.8.0" + val slf4j = "2.0.5" + val log4j = "2.20.0" + val logback = "1.2.11" + val scalatest = "3.2.15" + val circe = "0.14.1" } diff --git a/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala b/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala index 8452ade1..6db62df9 100644 --- a/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala +++ b/src/main/scala/com/github/plume/oss/JimpleAst2Database.scala @@ -3,7 +3,6 @@ package com.github.plume.oss import better.files.File import com.github.plume.oss.drivers.IDriver import com.github.plume.oss.passes.base.AstCreationPass -import com.github.plume.oss.passes.incremental.{PlumeDiffPass, PlumeHashPass} import io.joern.jimple2cpg.Jimple2Cpg.language import io.joern.jimple2cpg.passes.SootAstCreationPass import io.joern.jimple2cpg.{Config, Jimple2Cpg} @@ -45,7 +44,8 @@ class JimpleAst2Database(driver: IDriver, sootOnlyBuild: Boolean = false) { src, tmpDir, isClass = e => e.extension.contains(".class"), - isArchive = e => e.extension.exists(archiveFileExtensions.contains), + isArchive = e => e.isZipFile, + isConfigFile = e => e.isConfigFile, false, 0 ) @@ -72,11 +72,10 @@ class JimpleAst2Database(driver: IDriver, sootOnlyBuild: Boolean = false) { val input = File(config.inputPath) configureSoot(config, tmpDir) - val sourceFileNames = loadClassFiles(input, tmpDir) + val codeToProcess = loadClassFiles(input, tmpDir) logger.info("Loading classes to soot") // Load classes into Soot - val codeToProcess = new PlumeDiffPass(tmpDir.pathAsString, sourceFileNames, driver).createAndApply().toList sootLoad(codeToProcess) Scene.v().loadNecessaryClasses() logger.info(s"Loaded ${Scene.v().getApplicationClasses.size()} classes") @@ -85,7 +84,6 @@ class JimpleAst2Database(driver: IDriver, sootOnlyBuild: Boolean = false) { // Project Soot classes val astCreator = new AstCreationPass(codeToProcess.map(_.file.pathAsString), driver, tmpDir) astCreator.createAndApply() - new PlumeHashPass(driver).createAndApply() } } diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala index 41aac56f..a1b2ebee 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentWriterPass.scala @@ -18,7 +18,7 @@ abstract class PlumeConcurrentWriterPass[T <: AnyRef](driver: IDriver) { @volatile var nDiffT = -1 - def generateParts(): Array[_ <: AnyRef] + def generateParts(): Array[? <: AnyRef] // main function: add desired changes to builder def runOnPart(builder: DiffGraphBuilder, part: T): Unit diff --git a/src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala deleted file mode 100644 index 07cceb24..00000000 --- a/src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.github.plume.oss.passes.incremental - -import better.files.File -import com.github.plume.oss.PlumeStatistics -import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.util.HashUtil -import io.joern.jimple2cpg.util.ProgramHandlingUtil.ClassFile -import io.shiftleft.codepropertygraph.generated.{NodeTypes, PropertyNames} -import org.slf4j.{Logger, LoggerFactory} - -import java.io.File as JFile - -/** This pass finds all files that have changed in the database when compared to the currently loaded and will remove - * all related information to the files which have changed from the database. - */ -class PlumeDiffPass(pathRoot: String, classFiles: List[ClassFile], driver: IDriver) { - - import PlumeDiffPass.* - - private val classHashes: Map[String, String] = - driver - .propertyFromNodes(NodeTypes.FILE, PropertyNames.NAME, PropertyNames.HASH, "") - .flatMap { m => - val name = m.getOrElse(PropertyNames.NAME, "") - val hash = m.getOrElse(PropertyNames.HASH, "") - if (hash != null) { - Some(stripTempDirPrefix(name.toString) -> hash.toString) - } else { - None - } - } - .toMap - - /** Removes the temporary directory prefix which differs between extractions. - * @param filePath - * the path at which the file is located. - * @return - * the name of the file from a project root level. - */ - private def stripTempDirPrefix(filePath: String): String = - filePath.replaceAll(s"$pathRoot${JFile.separator}", "") - - /** Find and delete changed source files. - */ - def createAndApply(): Seq[ClassFile] = { - val changedFiles = classFiles - .filter { f => - classHashes.get(stripTempDirPrefix(f.file.pathAsString)) match { - case Some(hash) => HashUtil.getFileHash(f.file) != hash - case None => false // New files - } - } - - if (changedFiles.nonEmpty) { - logger.debug(s"Detected changes in the following files: ${changedFiles.mkString(", ")}") - PlumeStatistics.time( - PlumeStatistics.TIME_REMOVING_OUTDATED_GRAPH, { - driver.removeSourceFiles(changedFiles.map(_.file.pathAsString): _*) - } - ) - } - val newFiles = classFiles - .filterNot { f => classHashes.contains(stripTempDirPrefix(f.file.pathAsString)) } - - changedFiles ++ newFiles - } - -} - -object PlumeDiffPass { - val logger: Logger = LoggerFactory.getLogger(PlumeDiffPass.getClass) -} diff --git a/src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala deleted file mode 100644 index 4b960f94..00000000 --- a/src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.github.plume.oss.passes.incremental - -import better.files.File as BetterFile -import com.github.plume.oss.drivers.{IDriver, OverflowDbDriver} -import com.github.plume.oss.passes.PlumeConcurrentWriterPass -import com.github.plume.oss.util.HashUtil -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{File, NewFile} -import io.shiftleft.codepropertygraph.generated.{NodeTypes, PropertyNames} -import io.shiftleft.semanticcpg.language.* -import org.slf4j.{Logger, LoggerFactory} -import overflowdb.BatchedUpdate.DiffGraphBuilder -import overflowdb.{DetachedNodeGeneric, Edge, Graph, Node, Property, PropertyKey} - -import java.util -import java.util.Optional -import scala.util.{Failure, Success, Try} - -case class StoredFile(id: Long, name: String) - -/** Performs hash calculations on the files represented by the FILE nodes. This is - */ -class PlumeHashPass(driver: IDriver) extends PlumeConcurrentWriterPass[StoredFile](driver) { - - import PlumeHashPass.* - - /** We only hash application files who have not been hashed before. Any newly added files will have fresh FILE nodes - * without a defined HASH property. - */ - override def generateParts(): Array[StoredFile] = { - driver - .propertyFromNodes(NodeTypes.FILE, "id", PropertyNames.NAME, PropertyNames.HASH, PropertyNames.IS_EXTERNAL) - .filterNot(_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean) - .filter(_.getOrElse(PropertyNames.HASH, "").toString.isBlank) - .map(f => StoredFile(f("id").toString.toLong, f(PropertyNames.NAME).toString)) - .toArray - } - - /** Use the information in the given file node to find the local file and store its hash locally. - */ - override def runOnPart(diffGraph: DiffGraphBuilder, part: StoredFile): Unit = { - val localDiff = new DiffGraphBuilder - Try(HashUtil.getFileHash(BetterFile(part.name))) match { - case Failure(exception) => - logger.warn(s"Unable to generate hash for file at $part", exception) - case Success(fileHash) => - val node = AnonymousNode(part.id, NodeTypes.FILE) - localDiff.setNodeProperty(node, PropertyNames.HASH, fileHash) - } - diffGraph.absorb(localDiff) - } - -} - -object PlumeHashPass { - val logger: Logger = LoggerFactory.getLogger(PlumeHashPass.getClass) -} - -class AnonymousNode(val id: Long, val label: String) extends Node { - override def both(edgeLabels: String*): util.Iterator[Node] = null - - override def addEdgeImpl(label: String, inNode: Node, keyValues: Any*): Edge = null - - override def addEdgeImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Edge = null - - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: Any*): Unit = {} - - override def addEdgeSilentImpl(label: String, inNode: Node, keyValues: util.Map[String, AnyRef]): Unit = {} - - override def out(): util.Iterator[Node] = null - - override def out(edgeLabels: String*): util.Iterator[Node] = null - - override def in(): util.Iterator[Node] = null - - override def in(edgeLabels: String*): util.Iterator[Node] = null - - override def both(): util.Iterator[Node] = null - - override def outE(): util.Iterator[Edge] = null - - override def outE(edgeLabels: String*): util.Iterator[Edge] = null - - override def inE(): util.Iterator[Edge] = null - - override def inE(edgeLabels: String*): util.Iterator[Edge] = null - - override def bothE(): util.Iterator[Edge] = null - - override def bothE(edgeLabels: String*): util.Iterator[Edge] = null - - override def graph(): Graph = null - - override def propertyKeys(): util.Set[String] = null - - override def property(key: String): AnyRef = null - - override def property[A](key: PropertyKey[A]): A = null.asInstanceOf[A] - - override def propertyOption[A](key: PropertyKey[A]): Optional[A] = null - - override def propertyOption(key: String): Optional[AnyRef] = null - - override def propertiesMap(): util.Map[String, AnyRef] = null - - override def setPropertyImpl(key: String, value: Any): Unit = {} - - override def setPropertyImpl[A](key: PropertyKey[A], value: A): Unit = {} - - override def setPropertyImpl(property: Property[_]): Unit = {} - - override def removePropertyImpl(key: String): Unit = {} - - override def removeImpl(): Unit = {} -} diff --git a/src/test/resources/diff/Bar1.java b/src/test/resources/diff/Bar1.java deleted file mode 100644 index a551858f..00000000 --- a/src/test/resources/diff/Bar1.java +++ /dev/null @@ -1,9 +0,0 @@ -package diff; - -class Bar { - - public static int bar(int x, int y) { - return x + y; - } - -} \ No newline at end of file diff --git a/src/test/resources/diff/Foo1.java b/src/test/resources/diff/Foo1.java deleted file mode 100644 index 0a492de9..00000000 --- a/src/test/resources/diff/Foo1.java +++ /dev/null @@ -1,10 +0,0 @@ -package diff; - -class Foo { - - public static void foo(int a) { - var b = 1; - Bar.bar(a, b); - } - -} \ No newline at end of file diff --git a/src/test/resources/diff/Foo2.java b/src/test/resources/diff/Foo2.java deleted file mode 100644 index ec2404f3..00000000 --- a/src/test/resources/diff/Foo2.java +++ /dev/null @@ -1,10 +0,0 @@ -package diff; - -class Foo { - - public static void foo(int a) { - var b = 3; - Bar.bar(a, b); - } - -} \ No newline at end of file diff --git a/src/test/resources/unpacking/HelloWorld.jar b/src/test/resources/unpacking/HelloWorld.jar deleted file mode 100644 index 56ccb26f..00000000 Binary files a/src/test/resources/unpacking/HelloWorld.jar and /dev/null differ diff --git a/src/test/scala/com/github/plume/oss/DiffTests.scala b/src/test/scala/com/github/plume/oss/DiffTests.scala deleted file mode 100644 index 0e206b72..00000000 --- a/src/test/scala/com/github/plume/oss/DiffTests.scala +++ /dev/null @@ -1,127 +0,0 @@ -package com.github.plume.oss - -import com.github.plume.oss.drivers.OverflowDbDriver -import io.joern.dataflowengineoss.queryengine.QueryEngineStatistics -import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes, Operators, PropertyNames} -import io.shiftleft.semanticcpg.language._ -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec -import org.slf4j.LoggerFactory - -import java.io.{File, FileInputStream, FileOutputStream} -import java.nio.file.{Files, Paths} -import scala.util.Using - -/** Used to make sure that incremental change detection works correctly. - */ -class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { - - private val logger = LoggerFactory.getLogger(classOf[DiffTests]) - - private def getTestResource(dir: String): File = { - val resourceURL = getClass.getResource(dir) - if (resourceURL == null) throw new NullPointerException("Unable to obtain test resource") - new File(resourceURL.getFile) - } - - private def rewriteFileContents(tgt: File, incoming: File): File = { - if (!tgt.exists()) tgt.createNewFile() - Using.resources(new FileOutputStream(tgt, false), new FileInputStream(incoming)) { case (fos, fis) => - val buf = new Array[Byte](4096) - Iterator - .continually(fis.read(buf)) - .takeWhile(_ != -1) - .foreach(fos.write(buf, 0, _)) - } - new File(tgt.getAbsolutePath) - } - - private val foo1: File = getTestResource("/diff/Foo1.java") - private val foo2: File = getTestResource("/diff/Foo2.java") - private val bar1: File = getTestResource("/diff/Bar1.java") - private val sandboxDir: File = Files.createTempDirectory("plume").toFile - private val foo = new File(s"${sandboxDir.getAbsolutePath}${File.separator}Foo.java") - private val bar = new File(s"${sandboxDir.getAbsolutePath}${File.separator}Bar.java") - private var driver = new OverflowDbDriver() - private var cpg: Option[Cpg] = None - private val storage = Some("./cpg_test.odb") - - override def beforeAll(): Unit = { - rewriteFileContents(foo, foo1) - rewriteFileContents(bar, bar1) - JavaCompiler.compileJava(foo, bar) - driver.clear() - driver.close() - driver = new OverflowDbDriver(storage) - cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false)) - sandboxDir.listFiles().foreach(_.delete()) - } - - override def afterAll(): Unit = { - driver.clear() - driver.close() - Paths.get(storage.get).toFile.delete() - } - - "should have all necessary nodes on first pass" in { - driver.propertyFromNodes(NodeTypes.TYPE_DECL).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.METHOD).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.TYPE, PropertyNames.FULL_NAME).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.NAMESPACE_BLOCK, PropertyNames.FULL_NAME).size shouldBe 3 - driver - .propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.FULL_NAME) - .size shouldBe 10 - driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 2 - - val List(barM, fooM) = driver - .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) - .filter(!_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean) - .sortWith { case (x, y) => - x(PropertyNames.FULL_NAME).toString < y(PropertyNames.FULL_NAME).toString - } - fooM.get(PropertyNames.FULL_NAME) shouldBe Some("diff.Foo") - barM.get(PropertyNames.FULL_NAME) shouldBe Some("diff.Bar") - val List(lb) = driver - .propertyFromNodes(NodeTypes.LITERAL, PropertyNames.CODE) - lb.get(PropertyNames.CODE) shouldBe Some("1") - val paramA = driver - .propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.CODE) - .filter(_(PropertyNames.CODE) == "int a") - paramA.size shouldBe 1 - } - - "should update Foo on file changes" in { - rewriteFileContents(bar, bar1) - rewriteFileContents(foo, foo2) - JavaCompiler.compileJava(foo, bar) - cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false)) - - // Check the correct numbers of nodes are present - driver.propertyFromNodes(NodeTypes.TYPE_DECL).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.METHOD).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.TYPE, PropertyNames.FULL_NAME).size shouldBe 7 - driver.propertyFromNodes(NodeTypes.NAMESPACE_BLOCK, PropertyNames.FULL_NAME).size shouldBe 3 - driver - .propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.FULL_NAME) - .size shouldBe 10 - driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 2 - - val List(barM, fooM) = driver - .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) - .filter(!_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean) - .sortWith { case (x, y) => - x(PropertyNames.FULL_NAME).toString < y(PropertyNames.FULL_NAME).toString - } - fooM.get(PropertyNames.FULL_NAME) shouldBe Some("diff.Foo") - barM.get(PropertyNames.FULL_NAME) shouldBe Some("diff.Bar") - val List(lb) = driver - .propertyFromNodes(NodeTypes.LITERAL, PropertyNames.CODE) - lb.get(PropertyNames.CODE) shouldBe Some("3") - val paramA = driver - .propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.CODE) - .filter(_(PropertyNames.CODE) == "int a") - paramA.size shouldBe 1 - } - -} diff --git a/src/test/scala/com/github/plume/oss/SourceSupportTests.scala b/src/test/scala/com/github/plume/oss/SourceSupportTests.scala deleted file mode 100644 index 8cd5c3a2..00000000 --- a/src/test/scala/com/github/plume/oss/SourceSupportTests.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.github.plume.oss - -import com.github.plume.oss.drivers.OverflowDbDriver -import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes, PropertyNames} -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec -import org.slf4j.LoggerFactory - -import java.io.File -import java.nio.file.{Files, Paths} - -class SourceSupportTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { - - private def getTestResource(dir: String): File = { - val resourceURL = getClass.getResource(dir) - if (resourceURL == null) throw new NullPointerException("Unable to obtain test resource") - new File(resourceURL.getFile) - } - - private val foo: File = getTestResource("/source_support/Foo.java") - private val bar: File = getTestResource("/source_support/Bar.java") - - private var driver = new OverflowDbDriver() - private var cpg: Option[Cpg] = None - private val storage = Some("./cpg_test.odb") - private val sandboxDir: File = Files.createTempDirectory("plume").toFile - - override def beforeAll(): Unit = { - Paths.get(storage.get).toFile.delete() - driver.clear() - driver.close() - driver = new OverflowDbDriver(storage) - cpg = Some(new Jimple2Cpg().createCpg(foo.getParent, driver = driver)) - sandboxDir.listFiles().foreach(_.delete()) - } - - override def afterAll(): Unit = { - driver.clear() - driver.close() - Paths.get(storage.get).toFile.delete() - } - - "should accept Java source files" in { - val List(barM, fooM) = driver - .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.NAME, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) - .filter(!_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean) - .sortWith { case (x, y) => - x(PropertyNames.FULL_NAME).toString < y(PropertyNames.FULL_NAME).toString - } - fooM.get(PropertyNames.FULL_NAME) shouldBe Some("Foo") - barM.get(PropertyNames.FULL_NAME) shouldBe Some("Bar") - fooM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false) - barM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false) - } - -} diff --git a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala b/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala index ac9e008d..fe001ecf 100644 --- a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala +++ b/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala @@ -1,50 +1,42 @@ package com.github.plume.oss.testfixtures -import com.github.plume.oss.{Jimple2Cpg, PlumeStatistics} +import com.github.plume.oss.{JimpleAst2Database, PlumeStatistics} import com.github.plume.oss.drivers.OverflowDbDriver import com.github.plume.oss.JavaCompiler.compileJava -import io.joern.x2cpg.testfixtures.{CodeToCpgFixture, LanguageFrontend} +import io.joern.jimple2cpg.testfixtures.JimpleCodeToCpgFixture +import io.joern.jimple2cpg.Config +import io.joern.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, DefaultTestCpg} import io.shiftleft.codepropertygraph.Cpg import org.slf4j.LoggerFactory import java.io.{File, PrintWriter} -import java.nio.file.Files +import java.nio.file.Path import scala.util.Using -class PlumeFrontend(val _driver: Option[OverflowDbDriver]) extends LanguageFrontend { +trait PlumeFrontend(val _driver: Option[OverflowDbDriver]) extends LanguageFrontend { + + private val logger = LoggerFactory.getLogger(classOf[PlumeFrontend]) + override val fileSuffix: String = ".java" - private val logger = LoggerFactory.getLogger(classOf[PlumeFrontend]) val driver: OverflowDbDriver = _driver match { case Some(d) => d case None => new OverflowDbDriver() } - override val fileSuffix: String = ".java" override def execute(sourceCodeFile: File): Cpg = { PlumeStatistics.reset() - new Jimple2Cpg().createCpg(sourceCodeFile.getAbsolutePath, driver = driver) + new JimpleAst2Database(driver).createAst(Config().withInputPath(sourceCodeFile.getAbsolutePath)) logger.info(s"Plume statistics from last test: ${PlumeStatistics.results()}") Cpg(driver.cpg.graph) } } -class Jimple2CpgFixture(_driver: Option[OverflowDbDriver] = None) extends CodeToCpgFixture(new PlumeFrontend(_driver)) { - - val driver: OverflowDbDriver = frontend.asInstanceOf[PlumeFrontend].driver +class PlumeTestCpg(_driver: Option[OverflowDbDriver]) extends DefaultTestCpg with PlumeFrontend(_driver) { - override def passes(cpg: Cpg): Unit = {} - - override def writeCodeToFile(sourceCode: String): File = { - val tmpDir = Files.createTempDirectory("semanticcpgtest").toFile - tmpDir.deleteOnExit() - val codeFile = File.createTempFile("Test", frontend.fileSuffix, tmpDir) - Using.resource(new PrintWriter(codeFile)) { pw => pw.write(sourceCode) } - try { - compileJava(codeFile) - } finally { - codeFile.delete() - } - tmpDir + override protected def codeDirPreProcessing(rootFile: Path, codeFiles: List[Path]): Unit = { + val sourceFiles = codeFiles.map(_.toFile).filter(_.getName.endsWith(".java")) + if (sourceFiles.nonEmpty) JimpleCodeToCpgFixture.compileJava(rootFile, sourceFiles) } - } + +class Jimple2CpgFixture(_driver: Option[OverflowDbDriver] = None) extends Code2CpgFixture(() => PlumeTestCpg(_driver)) diff --git a/src/test/scala/com/github/plume/oss/unpacking/JarUnpackingTests.scala b/src/test/scala/com/github/plume/oss/unpacking/JarUnpackingTests.scala deleted file mode 100644 index f4534db2..00000000 --- a/src/test/scala/com/github/plume/oss/unpacking/JarUnpackingTests.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.github.plume.oss.unpacking - -import com.github.plume.oss.Jimple2Cpg -import io.joern.jimple2cpg.util.ProgramHandlingUtil -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.semanticcpg.language.{toMethodTraversalExtGen, toNodeTypeStarters, toTypeDeclTraversalExtGen} -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.must.Matchers -import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper -import org.scalatest.wordspec.AnyWordSpec - -import scala.reflect.io.File -import scala.util.{Failure, Success, Try} - -class JarUnpackingTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { - - var cpg: Cpg = _ - - override protected def beforeAll(): Unit = { - super.beforeAll() - Try(getClass.getResource("/unpacking")) match { - case Success(x) => - cpg = new Jimple2Cpg().createCpg(x.getPath) - case Failure(x: Throwable) => - fail("Unable to obtain test resources.", x) - } - } - - "should extract files and clean up temp directory" in { - Try(getClass.getResource("/unpacking")) match { - case Success(x) => - val fs = File(x.getPath).toDirectory.walk.toSeq - val cs = File(ProgramHandlingUtil.getUnpackingDir.toString).toDirectory.walk.toSeq - fs.filter(_.name.contains(".jar")).map(_.name).toList shouldBe List("HelloWorld.jar") - cs.count(_.name.contains(".class")) shouldBe 0 - case Failure(x: Throwable) => - fail("Unable to view test repository", x) - } - } - - "should reflect the correct package order" in { - val List(foo) = cpg.typeDecl.fullNameExact("Foo").l - foo.name shouldBe "Foo" - - val List(bar) = cpg.typeDecl.fullNameExact("pac.Bar").l - bar.name shouldBe "Bar" - - cpg.method.filterNot(_.isExternal).fullName.toSet shouldBe Set( - "Foo.:void()", - "Foo.add:int(int,int)", - "pac.Bar.sub:int(int,int)", - "pac.Bar.:void()" - ) - } - -}