From 4c5afdadb92d2e5356452120ee0f16a464df75c3 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Tue, 23 Jul 2024 08:49:49 +0200 Subject: [PATCH] flatgraph port (#344) * port to flatgraph * upgrade to latest --- build.sbt | 6 ++-- project/build.properties | 2 +- .../js2cpg/astcreation/AstCreator.scala | 6 ++-- .../js2cpg/astcreation/AstEdgeBuilder.scala | 2 +- .../js2cpg/astcreation/AstNodeBuilder.scala | 2 +- .../js2cpg/astcreation/NewCompositeNode.scala | 17 +++++----- .../js2cpg/passes/AstCreationPass.scala | 8 ++--- .../shiftleft/js2cpg/passes/ConfigPass.scala | 8 ++--- .../io/shiftleft/js2cpg/io/ExcludeTest.scala | 2 +- .../js2cpg/io/PrivateModulesTest.scala | 8 ++--- .../js2cpg/passes/AbstractPassTest.scala | 5 ++- .../js2cpg/passes/BuiltinTypesPassTest.scala | 4 +-- .../js2cpg/passes/CfgCreationPassTest.scala | 21 ++++++------ .../js2cpg/passes/ConfigPassTest.scala | 32 +++++++++---------- .../js2cpg/passes/JsMetaDataPassTest.scala | 11 +++---- .../TranspilationRunnerTest.scala | 24 +++++++------- 16 files changed, 77 insertions(+), 81 deletions(-) diff --git a/build.sbt b/build.sbt index 2983f561..567f6b9b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,5 @@ -val cpgVersion = "1.6.11" -val joernVersion = "2.0.335" +val cpgVersion = "1.7.3" +val joernVersion = "4.0.8" val gitCommitString = SettingKey[String]("gitSha") @@ -16,7 +16,7 @@ Global / excludeLintKeys += Fast / configuration Global / excludeLintKeys += gitCommitString lazy val commonSettings = Seq( - scalaVersion := "3.4.1", + scalaVersion := "3.4.2", organization := "io.shiftleft", scalacOptions ++= Seq("-Xtarget:8"), resolvers ++= Seq( diff --git a/project/build.properties b/project/build.properties index 04267b14..ee4c672c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.9 +sbt.version=1.10.1 diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala index b1e34ddb..5fa389fa 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala @@ -44,6 +44,7 @@ import com.oracle.js.parser.ir.{ WithNode } import com.oracle.js.parser.ir.LiteralNode.ArrayLiteralNode +import flatgraph.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.nodes.{ NewBlock, NewCall, @@ -64,7 +65,6 @@ import io.shiftleft.js2cpg.datastructures.scope._ import io.shiftleft.js2cpg.passes.{Defines, EcmaBuiltins, PassHelpers} import io.shiftleft.js2cpg.passes.PassHelpers.ParamNodeInitKind import io.shiftleft.js2cpg.parser.{GeneralizingAstVisitor, JsSource} -import overflowdb.BatchedUpdate.DiffGraphBuilder import org.slf4j.LoggerFactory import scala.collection.mutable @@ -429,7 +429,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: astNodeBuilder.createTypeNode(methodName, methodFullName) val astParentType = parentNodeId.label - val astParentFullName = parentNodeId.properties("FULL_NAME").toString + val astParentFullName = parentNodeId.propertiesMap.get("FULL_NAME").toString val functionTypeDeclId = astNodeBuilder.createTypeDeclNode(methodName, methodFullName, astParentType, astParentFullName, Some(Defines.Any)) @@ -457,7 +457,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: // need to be resolved first, we for now dont handle the class // hierarchy. val astParentType = methodAstParentStack.head.label - val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString + val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString val typeDeclId = astNodeBuilder.createTypeDeclNode(typeName, typeFullName, astParentType, astParentFullName, inheritsFrom = None) diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala index 41e179c4..bb366820 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala @@ -1,9 +1,9 @@ package io.shiftleft.js2cpg.astcreation +import flatgraph.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.js2cpg.datastructures.OrderTracker -import overflowdb.BatchedUpdate.DiffGraphBuilder import org.slf4j.LoggerFactory class AstEdgeBuilder(private val diffGraph: DiffGraphBuilder) { diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala index 6ccbdb63..8bb89439 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala @@ -1,6 +1,7 @@ package io.shiftleft.js2cpg.astcreation import com.oracle.js.parser.ir._ +import flatgraph.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators} import io.shiftleft.js2cpg.datastructures.{LineAndColumn, OrderTracker} @@ -8,7 +9,6 @@ import io.shiftleft.js2cpg.datastructures.scope.{MethodScope, Scope} import io.shiftleft.js2cpg.passes.Defines import io.shiftleft.js2cpg.parser.JsSource import io.shiftleft.js2cpg.parser.JsSource.shortenCode -import overflowdb.BatchedUpdate.DiffGraphBuilder class AstNodeBuilder( private val diffGraph: DiffGraphBuilder, diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala index 80cac920..25375447 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala @@ -1,14 +1,14 @@ package io.shiftleft.js2cpg.astcreation -import io.shiftleft.codepropertygraph.generated.nodes.NewNode +import flatgraph.BatchedUpdateInterface +import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode} +import java.util import scala.collection.mutable.ListBuffer -class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode { +class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode(nodeKind = -1) { override def label: String = "COMPOSITE" - override def properties: Map[String, Any] = ??? // we do not need this - override def canEqual(that: Any): Boolean = that.isInstanceOf[NewCompositeNode] @@ -29,8 +29,9 @@ class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNod underlying.foreach(func) } - def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this - - def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this - + override def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this + override def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this + override def propertiesMap: util.Map[String, Any] = ??? // we do not need this + override def countAndVisitProperties(interface: BatchedUpdateInterface): Unit = ??? + override type StoredNodeType = StoredNode } diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala index 0b77da5f..145d7b86 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala @@ -4,12 +4,12 @@ import java.nio.file.Path import better.files.File import com.oracle.js.parser.Source import com.oracle.js.parser.ir.FunctionNode -import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.js2cpg.core.Report import io.shiftleft.js2cpg.astcreation.AstCreator import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks} import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource} -import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.passes.ForkJoinParallelCpgPass import org.slf4j.LoggerFactory import io.shiftleft.js2cpg.utils.SourceWrapper._ import io.shiftleft.js2cpg.utils.TimeUtils @@ -20,7 +20,7 @@ import scala.util.{Failure, Success, Try} * in parallel. */ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, report: Report) - extends ConcurrentWriterCpgPass[(Path, Path)](cpg) { + extends ForkJoinParallelCpgPass[(Path, Path)](cpg) { private val logger = LoggerFactory.getLogger(getClass) @@ -41,7 +41,7 @@ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, rep parseResult.map { case (parseResult, usedIdentNodes) => val (result, duration) = { - TimeUtils.time(generateCpg(parseResult, new DiffGraphBuilder, usedIdentNodes)) + TimeUtils.time(generateCpg(parseResult, Cpg.newDiffGraphBuilder, usedIdentNodes)) } val path = parseResult.jsSource.originalFilePath result match { diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala index 31d07a23..3d8410ad 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala @@ -1,19 +1,19 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile import io.shiftleft.js2cpg.core.Report import io.shiftleft.js2cpg.io.FileDefaults import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.js2cpg.utils.TimeUtils -import io.shiftleft.passes.ConcurrentWriterCpgPass +import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.utils.IOUtils import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Path class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report) - extends ConcurrentWriterCpgPass[(Path, Path)](cpg) { + extends ForkJoinParallelCpgPass[(Path, Path)](cpg) { private val logger: Logger = LoggerFactory.getLogger(getClass) @@ -37,7 +37,7 @@ class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report) val fileName = relativeFile.toString val content = fileContent(filePath) val (result, time) = TimeUtils.time { - val localDiff = new DiffGraphBuilder + val localDiff = Cpg.newDiffGraphBuilder logger.debug(s"Adding file '$relativeFile' as config file.") val configNode = NewConfigFile().name(fileName).content(content.mkString("\n")) report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true, isConfig = true) diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala index f7b7aa88..451c6e8d 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala @@ -40,7 +40,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec defaultArgs.map(_.toArg).toArray ) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false) fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator)) cpg.close() diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index 4110640c..e5bd827b 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -24,7 +24,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false) fileNames(cpg) should contain allElementsOf Set( s"@privateA${java.io.File.separator}a.js", @@ -52,7 +52,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { ) ) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false) fileNames(cpg) should contain allElementsOf Set( s"@privateA${java.io.File.separator}a.js", @@ -82,7 +82,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { ) ) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false) fileNames(cpg) should contain only s"@privateB${java.io.File.separator}b.js" cpg.close() @@ -98,7 +98,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false) fileNames(cpg) should contain only "index.js" cpg.close() diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala index 05476222..c2ba885e 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala @@ -7,8 +7,6 @@ import io.shiftleft.codepropertygraph.generated.nodes.Dependency import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.js2cpg.core.Report import io.shiftleft.semanticcpg.language.* -import overflowdb.Node -import overflowdb.traversal.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -18,7 +16,8 @@ abstract class AbstractPassTest extends AnyWordSpec with Matchers { protected abstract class Fixture - protected def getDependencies(cpg: Cpg): Traversal[Dependency] = cpg.dependency + protected def getDependencies(cpg: Cpg): Iterator[Dependency] = + cpg.dependency protected object AstFixture extends Fixture { def apply(code: String)(f: Cpg => Unit): Unit = { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala index 35304737..d0de1884 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala @@ -1,12 +1,12 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language._ class BuiltinTypesPassTest extends AbstractPassTest { "BuiltinTypesPass" should { - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty new BuiltinTypesPass(cpg).createAndApply() "create a '' NamespaceBlock" in { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala index e7b61f29..daeb6e62 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala @@ -1,17 +1,14 @@ package io.shiftleft.js2cpg.passes import better.files.File -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated._ +import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.js2cpg.core.Report -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.controlflow.CfgCreationPass -import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg._ -import io.shiftleft.codepropertygraph.generated.nodes.AstNodeBase -import io.shiftleft.codepropertygraph.generated.nodes.Method +import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.* +import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeBase, CfgNode, Method} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import overflowdb.Node class CfgCreationPassTest extends AnyWordSpec with Matchers { @@ -1290,7 +1287,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers { private class CfgFixture(code: String) { - private val cpg: Cpg = Cpg.emptyCpg + private val cpg: Cpg = Cpg.empty File.usingTemporaryDirectory("js2cpgCfgIntegrationTest") { workspace => val file = workspace / "test.js" @@ -1300,7 +1297,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers { new CfgCreationPass(cpg).createAndApply() } - private def matchCode(node: Node, code: String): Boolean = node match { + private def matchCode(node: CfgNode, code: String): Boolean = node match { case m: Method => m.name == code case astNode: AstNodeBase => astNode.code == code case _ => false @@ -1325,7 +1322,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers { } .lift(index) .getOrElse(fail(s"No node found for code = '$code' and index '$index'!")) - node.property(PropertyNames.CODE).toString + node.code }.toSet } @@ -1337,14 +1334,14 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers { def succOf(code: String, index: Int): Set[String] = { val nodes = cpg.method.ast.isCfgNode.collect { case node if matchCode(node, code) => node }.l val node = nodes.lift(index).getOrElse(fail(s"No node found for code = '$code' and index '$index'!")) - val successors = node._cfgOut.map(_.property(PropertyNames.CODE).toString) + val successors = node._cfgOut.cast[CfgNode].code successors.toSetImmutable } def succOf(code: String, nodeType: String): Set[String] = { val nodes = cpg.method.ast.isCfgNode.label(nodeType).collectFirst { case node if matchCode(node, code) => node } val node = nodes.getOrElse(fail(s"No node found for code = '$code' and type '$nodeType'!")) - val successors = node._cfgOut.map(_.property(PropertyNames.CODE).toString) + val successors = node._cfgOut.cast[CfgNode].code successors.toSetImmutable } diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala index 1cc004ac..02e9e7c4 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala @@ -1,8 +1,8 @@ package io.shiftleft.js2cpg.passes import io.shiftleft.js2cpg.core.Report -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import better.files.File import io.shiftleft.js2cpg.io.FileDefaults import org.scalatest.matchers.should.Matchers @@ -18,12 +18,12 @@ class ConfigPassTest extends AnyWordSpec with Matchers { val fileB = dir / "b.vue" fileA.write("someCodeA();") fileB.write("someCodeB();") - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty val filenames = List((fileA.path, fileA.parent.path), (fileB.path, fileB.parent.path)) new ConfigPass(filenames, cpg, new Report()).createAndApply() - val List(configFileA) = cpg.configFile("a.vue").l - val List(configFileB) = cpg.configFile("b.vue").l + val List(configFileA) = cpg.configFile.name("a.vue").l + val List(configFileB) = cpg.configFile.name("b.vue").l configFileA.content shouldBe "someCodeA();" configFileB.content shouldBe "someCodeB();" } @@ -43,14 +43,14 @@ class ConfigPassTest extends AnyWordSpec with Matchers { fileB.write("b") fileC.write("c") - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty val filenames = List((fileA.path, fileA.parent.path), (fileB.path, fileB.parent.path), (fileC.path, fileC.parent.path)) new ConfigPass(filenames, cpg, new Report()).createAndApply() - val List(configFileA) = cpg.configFile("a.conf.js").l - val List(configFileB) = cpg.configFile("b.config.js").l - val List(configFileC) = cpg.configFile("c.json").l + val List(configFileA) = cpg.configFile.name("a.conf.js").l + val List(configFileB) = cpg.configFile.name("b.config.js").l + val List(configFileC) = cpg.configFile.name("c.json").l configFileA.content shouldBe "a" configFileB.content shouldBe "b" configFileC.content shouldBe "c" @@ -65,7 +65,7 @@ class ConfigPassTest extends AnyWordSpec with Matchers { fileA.write("x\n" * FileDefaults.NUM_LINES_THRESHOLD + 1) // too many lines fileB.write("x" * FileDefaults.LINE_LENGTH_THRESHOLD + 1) // line too long - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty val filenames = List((fileA.path, fileA.parent.path), (fileB.path, fileB.parent.path)) new ConfigPass(filenames, cpg, new Report()).createAndApply() @@ -85,12 +85,12 @@ class ConfigPassTest extends AnyWordSpec with Matchers { fileA.write("a") fileB.write("b") - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty val filenames = List((fileA.path, fileA.parent.path), (fileB.path, fileB.parent.path)) new ConfigPass(filenames, cpg, new Report()).createAndApply() - val List(configFileA) = cpg.configFile("a.html").l - val List(configFileB) = cpg.configFile("b.html").l + val List(configFileA) = cpg.configFile.name("a.html").l + val List(configFileB) = cpg.configFile.name("b.html").l configFileA.content shouldBe "a" configFileB.content shouldBe "b" } @@ -106,14 +106,14 @@ class ConfigPassTest extends AnyWordSpec with Matchers { fileA.write("-----BEGIN RSA PRIVATE KEY-----\n123456789\n-----END RSA PRIVATE KEY-----") val fileB = dir / "b.key" fileB.write("-----BEGIN SOME OTHER KEY-----\nthis is fine\n-----END SOME OTHER KEY-----") - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty val filenames = List((fileA.path, fileA.parent.path), (fileB.path, fileB.parent.path)) new PrivateKeyFilePass(filenames, cpg, new Report()).createAndApply() - val List(configFileA) = cpg.configFile("a.key").l + val List(configFileA) = cpg.configFile.name("a.key").l configFileA.content shouldBe "Content omitted for security reasons." - cpg.configFile("b.key") shouldBe empty + cpg.configFile.name("b.key") shouldBe empty } } diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala index 1f1b4974..9d132882 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala @@ -1,23 +1,22 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.Languages -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} +import io.shiftleft.semanticcpg.language.* import scala.jdk.CollectionConverters._ class JsMetaDataPassTest extends AbstractPassTest { "MetaDataPass" should { - val cpg = Cpg.emptyCpg + val cpg = Cpg.empty new JsMetaDataPass(cpg, "somehash", ".").createAndApply() "create exactly 1 node" in { - cpg.graph.V.asScala.size shouldBe 1 + cpg.graph.nodeCount shouldBe 1 } "create no edges" in { - cpg.graph.E.asScala.size shouldBe 0 + cpg.graph.edgeCount shouldBe 0 } "create a metadata node with correct language" in { diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index 80f68684..a89e2a12 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -1,7 +1,7 @@ package io.shiftleft.js2cpg.preprocessing import better.files.File -import io.shiftleft.codepropertygraph.Cpg +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.core.Js2CpgMain import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX @@ -16,7 +16,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { private def fileNames(cpg: Cpg): List[String] = cpg.file.name.l - private def callLineNumbers(cpg: Cpg): List[Integer] = cpg.call.lineNumber.l + private def callLineNumbers(cpg: Cpg): List[Int] = cpg.call.lineNumber.l private def callCodeFields(cpg: Cpg): List[String] = cpg.call.code.l @@ -74,7 +74,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List("foo.js") callCodeFields(cpg) should contain allElementsOf List( @@ -101,7 +101,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List("a.ts", "b.ts") callLineNumbers(cpg) should contain allElementsOf List(1, 1) @@ -154,7 +154,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( "index.js", @@ -178,7 +178,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel", "--with-tests")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( "a.ts", @@ -196,7 +196,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( "a.js", @@ -214,7 +214,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( "a.js", @@ -231,7 +231,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( s"src${java.io.File.separator}main.js", @@ -247,7 +247,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain theSameElementsAs List( s"src${java.io.File.separator}views${java.io.File.separator}AboutPage.vue", @@ -265,7 +265,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts", "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) fileNames(cpg) should contain only "test.ejs" callLineNumbers(cpg) should contain allElementsOf List(5, 7, 15, 16, 21) @@ -279,7 +279,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts", "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withStorage(cpgPath.path, persistOnClose = false) // Sadly, calling the pug transpiler via pug cli does not support source maps: fileNames(cpg) should contain only "test.js"