add dynamic analysis

.github/mypy/mypy.ini

@@ -1,5 +1,12 @@
 [mypy]
 
+# TODO(yelhamer): remove this once proto has been added
+# for the dynamic rendering
+exclude = (?x)(
+    ^capa/render/proto/__init__.py$
+    | ^tests/_test_proto.py$
+  )
+
 [mypy-halo.*]
 ignore_missing_imports = True
 

.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, "dynamic-feature-extraction" ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, "dynamic-feature-extraction" ]
 
 permissions: read-all
 
@@ -39,9 +39,9 @@ jobs:
     - name: Lint with ruff
       run: pre-commit run ruff
     - name: Lint with isort
-      run: pre-commit run isort
+      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
-      run: pre-commit run black
+      run: pre-commit run black --show-diff-on-failure
     - name: Lint with flake8
       run: pre-commit run flake8
     - name: Check types with mypy

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "rules"]
 	path = rules
 	url = ../capa-rules.git
+	branch = dynamic-syntax
 [submodule "tests/data"]
 	path = tests/data
 	url = ../capa-testfiles.git
+	branch = dynamic-feature-extractor

CHANGELOG.md

@@ -3,9 +3,15 @@
 ## master (unreleased)
 
 ### New Features
+- implement dynamic analysis via CAPE sandbox #48 #1535 @yelhamer
+- add call scope #771 @yelhamer
+- add process scope for the dynamic analysis flavor #1517 @yelhamer
+- Add thread scope for the dynamic analysis flavor #1517 @yelhamer
 
 ### Breaking Changes
 
+- remove the `SCOPE_*` constants in favor of the `Scope` enum #1764 @williballenthin
+
 ### New Rules (0)
 
 -

capa/engine.py

@@ -304,7 +304,7 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -
     other strategies can be imagined that match differently; implement these elsewhere.
     specifically, this routine does "top down" matching of the given rules against the feature set.
     """
-    results = collections.defaultdict(list)  # type: MatchResults
+    results: MatchResults = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)

capa/features/address.py

@@ -43,6 +43,79 @@ def __hash__(self):
         return int.__hash__(self)
 
 
+class ProcessAddress(Address):
+    """an address of a process in a dynamic execution trace"""
+
+    def __init__(self, pid: int, ppid: int = 0):
+        assert ppid >= 0
+        assert pid > 0
+        self.ppid = ppid
+        self.pid = pid
+
+    def __repr__(self):
+        return "process(%s%s)" % (
+            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
+            f"pid: {self.pid}",
+        )
+
+    def __hash__(self):
+        return hash((self.ppid, self.pid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) == (other.ppid, other.pid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) < (other.ppid, other.pid)
+
+
+class ThreadAddress(Address):
+    """addresses a thread in a dynamic execution trace"""
+
+    def __init__(self, process: ProcessAddress, tid: int):
+        assert tid >= 0
+        self.process = process
+        self.tid = tid
+
+    def __repr__(self):
+        return f"{self.process}, thread(tid: {self.tid})"
+
+    def __hash__(self):
+        return hash((self.process, self.tid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) == (other.process, other.tid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) < (other.process, other.tid)
+
+
+class DynamicCallAddress(Address):
+    """addesses a call in a dynamic execution trace"""
+
+    def __init__(self, thread: ThreadAddress, id: int):
+        assert id >= 0
+        self.thread = thread
+        self.id = id
+
+    def __repr__(self):
+        return f"{self.thread}, call(id: {self.id})"
+
+    def __hash__(self):
+        return hash((self.thread, self.id))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) == (other.thread, other.id)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) < (other.thread, other.id)
+
+
 class RelativeVirtualAddress(int, Address):
     """a memory address relative to a base address"""
 

capa/features/common.py

@@ -457,6 +457,17 @@ def evaluate(self, ctx, **kwargs):
 FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
+FORMAT_CAPE = "cape"
+STATIC_FORMATS = {
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_DOTNET,
+}
+DYNAMIC_FORMATS = {
+    FORMAT_CAPE,
+}
 FORMAT_FREEZE = "freeze"
 FORMAT_RESULT = "result"
 FORMAT_UNKNOWN = "unknown"