Skip to content

Commit ba0b1cc

Browse files
prabhuprabhu
authored
Automatically invoke cdxgen while importing code (#65)
* Automatically invoke cdxgen while importing code Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Use dynamic source Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
1 parent ba814fa commit ba0b1cc

File tree

12 files changed

+51
-27
lines changed

12 files changed

+51
-27
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,5 @@ chen.zip
5858
project/metals.sbt
5959
conda-out/
6060
notebooks/.ipynb_checkpoints/
61+
*.slices.json
62+

‎build.sbt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name := "chen"
22
ThisBuild / organization := "io.appthreat"
3-
ThisBuild / version := "2.0.3"
3+
ThisBuild / version := "2.0.4"
44
ThisBuild / scalaVersion := "3.3.1"
55

66
val cpgVersion = "1.0.0"

‎codemeta.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"downloadUrl": "https://github.com/AppThreat/chen",
88
"issueTracker": "https://github.com/AppThreat/chen/issues",
99
"name": "chen",
10-
"version": "2.0.3",
10+
"version": "2.0.4",
1111
"description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
1212
"applicationCategory": "code-analysis",
1313
"keywords": [

‎console/src/main/scala/io/appthreat/console/BridgeBase.scala

+3-4
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ trait ScriptExecution:
290290
*/
291291
private def importCpgCode(config: Config): List[String] =
292292
config.cpgToLoad.map { cpgFile =>
293-
"importCpg(\"" + cpgFile + "\")"
293+
"importAtom(\"" + cpgFile + "\")"
294294
}.toList ++ config.forInputPath.map { name =>
295295
s"""
296296
|openForInputPath(\"$name\")
@@ -340,9 +340,8 @@ trait PluginHandling:
340340

341341
val storeCode = if config.store then "save"
342342
else ""
343-
val runDataflow = if productName == "ocular" then "run.dataflow"
344-
else "run.ossdataflow"
345-
val argsString = argsStringFromConfig(config)
343+
val runDataflow = "run.ossdataflow"
344+
val argsString = argsStringFromConfig(config)
346345

347346
s"""
348347
| if (${config.overwrite} || !workspace.projectExists("$src")) {

‎console/src/main/scala/io/appthreat/console/ConsoleConfig.scala

+4-4
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ import scala.collection.mutable
1212
*/
1313
class InstallConfig(environment: Map[String, String] = sys.env):
1414

15-
/** determining the root path of the joern/ocular installation is rather complex unfortunately,
16-
* because we support a variety of use cases:
15+
/** determining the root path of the installation is rather complex unfortunately, because we
16+
* support a variety of use cases:
1717
* - running the installed distribution from the install dir
1818
* - running the installed distribution anywhere else on the system
19-
* - running a locally staged ocular/joern build (via `sbt stage` and then either `./joern`
20-
* or `cd platform/target/universal/stage; ./joern`)
19+
* - running a locally staged build (via `sbt stage` and then either `./chennai` or `cd
20+
* platform/target/universal/stage; ./chennai`)
2121
* - running a unit/integration test (note: the jars would be in the local cache, e.g. in
2222
* ~/.coursier/cache)
2323
*/

‎console/src/main/scala/io/appthreat/console/cpgcreation/AtomGenerator.scala

+17-2
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,38 @@ case class AtomGenerator(
1414
sliceMode: String = "reachables",
1515
slicesFile: String = "reachables.slices.json"
1616
) extends CpgGenerator:
17-
private lazy val command: String = "atom"
17+
private lazy val command: String = sys.env.getOrElse("ATOM_CMD", "atom")
18+
private lazy val cdxgenCommand: String = sys.env.getOrElse("CDXGEN_CMD", "cdxgen")
1819

1920
/** Generate an atom for the given input path. Returns the output path, or None, if no CPG was
2021
* generated.
2122
*/
2223
override def generate(inputPath: String, outputPath: String = "app.atom"): Try[String] =
24+
// If there is no bom.json file in the root directory, attempt to automatically invoke cdxgen
25+
val bomPath = File(inputPath) / "bom.json"
26+
if !bomPath.exists then
27+
val cdxLanguage = language.toLowerCase().replace("src", "")
28+
val arguments = Seq(
29+
"-t",
30+
cdxLanguage,
31+
"--deep",
32+
"-o",
33+
(File(inputPath) / "bom.json").pathAsString,
34+
inputPath
35+
)
36+
runShellCommand(cdxgenCommand, arguments)
2337
val arguments = Seq(
2438
sliceMode,
2539
"-s",
26-
slicesFile,
40+
(File(inputPath) / slicesFile).pathAsString,
2741
"--output",
2842
(File(inputPath) / outputPath).pathAsString,
2943
"--language",
3044
language,
3145
inputPath
3246
) ++ config.cmdLineParams
3347
runShellCommand(command, arguments).map(_ => (File(inputPath) / outputPath).pathAsString)
48+
end generate
3449

3550
override def isAvailable: Boolean = true
3651

‎console/src/test/scala/io/appthreat/console/LanguageHelperTests.scala

+7-7
Original file line numberDiff line numberDiff line change
@@ -26,47 +26,47 @@ class LanguageHelperTests extends AnyWordSpec with Matchers {
2626
}
2727

2828
"guess `JavaSrc` for a directory containing `.java`" in {
29-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
29+
File.usingTemporaryDirectory("chentests") { tmpDir =>
3030
val subdir = mkdir(tmpDir / "subdir")
3131
touch(subdir / "ServiceIdentifierComposerVisitorBasedStrategy.java")
3232
guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.JAVASRC)
3333
}
3434
}
3535

3636
"guess `Go` for a directory containing `Gopkg.lock`" in {
37-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
37+
File.usingTemporaryDirectory("chentests") { tmpDir =>
3838
val subdir = mkdir(tmpDir / "subdir")
3939
touch(subdir / "Gopkg.lock")
4040
guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.GOLANG)
4141
}
4242
}
4343

4444
"guess `Go` for a directory containing `Gopkg.toml`" in {
45-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
45+
File.usingTemporaryDirectory("chentests") { tmpDir =>
4646
val subdir = mkdir(tmpDir / "subdir")
4747
touch(subdir / "Gopkg.toml")
4848
guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.GOLANG)
4949
}
5050
}
5151

5252
"guess `Javascript` for a directory containing `package.json`" in {
53-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
53+
File.usingTemporaryDirectory("chentests") { tmpDir =>
5454
val subdir = mkdir(tmpDir / "subdir")
5555
touch(subdir / "package.json")
5656
guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.JSSRC)
5757
}
5858
}
5959

6060
"guess `C` for a directory containing .ll (LLVM) file" in {
61-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
61+
File.usingTemporaryDirectory("chentests") { tmpDir =>
6262
val subdir = mkdir(tmpDir / "subdir")
6363
touch(subdir / "foobar.ll")
6464
guessLanguage(tmpDir.pathAsString) shouldBe Some(Languages.LLVM)
6565
}
6666
}
6767

6868
"guess the language with the largest number of files" in {
69-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
69+
File.usingTemporaryDirectory("chentests") { tmpDir =>
7070
val subdir = mkdir(tmpDir / "subdir")
7171
touch(subdir / "source.c")
7272
touch(subdir / "source.java")
@@ -79,7 +79,7 @@ class LanguageHelperTests extends AnyWordSpec with Matchers {
7979
}
8080

8181
"not find anything for an empty directory" in {
82-
File.usingTemporaryDirectory("oculartests") { tmpDir =>
82+
File.usingTemporaryDirectory("chentests") { tmpDir =>
8383
guessLanguage(tmpDir.pathAsString) shouldBe None
8484
}
8585
}

‎macros/src/main/scala/io/appthreat/console/QueryDatabase.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ class QueryDatabase(
4444
// the namespace currently looks like `io.appthreat.scanners.c.CopyLoops`
4545
val namespaceParts = bundleNamespace.split('.')
4646
val language =
47-
if bundleNamespace.startsWith("io.appthreat.ocular.scanners") then
47+
if bundleNamespace.startsWith("io.appthreat.chen.scanners") then
4848
namespaceParts(4)
4949
else if namespaceParts.length > 3 then
5050
namespaceParts(3)

‎meta.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% set version = "2.0.3" %}
1+
{% set version = "2.0.4" %}
22

33
package:
44
name: chen

‎platform/src/main/scala/io/appthreat/chencli/console/ChenConsole.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class ChenConsole extends Console[ChenProject](new ChenWorkspaceLoader):
3333
.getOrElse(EngineContext())
3434

3535
def loadCpg(inputPath: String): Option[Cpg] =
36-
report("Deprecated. Please use `importCpg` instead")
36+
report("Deprecated. Please use `importAtom` instead")
3737
importCpg(inputPath)
3838

3939
override def applyDefaultOverlays(cpg: Cpg): Cpg =

‎platform/src/main/scala/io/appthreat/chencli/console/Predefined.scala

+12-4
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,24 @@ object Predefined:
2323
"""
2424
|def reachables(sinkTag: String, sourceTag: String, sourceTags: Array[String])(implicit atom: Cpg): Unit = {
2525
| try {
26-
| def source=atom.tag.name(sourceTag).parameter
26+
| val language = atom.metaData.language.l.head
2727
| def sources=sourceTags.map(t => atom.tag.name(t).parameter)
28-
| def sink=atom.ret.where(_.tag.name(sinkTag))
29-
| sink.df(source, sources).t
28+
| if language == Languages.JSSRC || language == Languages.JAVASCRIPT || language == Languages.PYTHON || language == Languages.PYTHONSRC
29+
| then
30+
| def source = atom.tag.name(sourceTag).call.argument
31+
| def sink = atom.tag.name(sinkTag).call.argument
32+
| sink.df(source, sources).t
33+
| else
34+
| def source=atom.tag.name(sourceTag).parameter
35+
| def sink=atom.ret.where(_.tag.name(sinkTag))
36+
| sink.df(source, sources).t
37+
| end if
3038
| } catch {
3139
| case exc: Exception =>
3240
| }
3341
|}
3442
|
35-
|def reachables(implicit atom: Cpg): Unit = reachables("framework-output", "framework-input", Array("api"))
43+
|def reachables(implicit atom: Cpg): Unit = reachables("framework-output", "framework-input", Array("api", "framework", "http", "cli-source", "library-call"))
3644
|
3745
|""".stripMargin
3846
)

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "appthreat-chen"
3-
version = "2.0.3"
3+
version = "2.0.4"
44
description = "Code Hierarchy Exploration Net (chen)"
55
authors = ["Team AppThreat <cloud@appthreat.com>"]
66
license = "Apache-2.0"

0 commit comments

Comments
 (0)