From 2daecce68322f8fa753224e59baf9157f4deb6d8 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 15:57:33 -0500 Subject: [PATCH 01/11] Added output directory option to Spoon analyser CLI --- .../java/christimperley/kaskara/Main.java | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index 2d53625..0384dfe 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -4,6 +4,10 @@ import com.fasterxml.jackson.databind.SerializationFeature; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -19,6 +23,11 @@ public class Main implements Callable { description = "The root source code directory for the project.") private String directory; + @CommandLine.Option(names = "-o", + defaultValue = ".", + description = "The directory to which results should be written.") + private String outputDirectory; + /** * Provides an entrypoint to the Kaskara Java analysis tool. * @@ -28,8 +37,30 @@ public static void main(String[] args) throws IOException { System.exit(new CommandLine(new Main()).execute(args)); } + /** + * Prepares the output directory by ensuring that it exists. + */ + private void prepareOutputDirectory() throws IOException { + this.outputDirectory = FileSystems.getDefault() + .getPath(this.outputDirectory) + .normalize() + .toAbsolutePath() + .toString(); + + Files.createDirectories(Paths.get(this.outputDirectory)); + System.out.printf("Output will be written to: %s%n", this.outputDirectory); + } + @Override public Integer call() throws IOException { + try { + this.prepareOutputDirectory(); + } catch (java.nio.file.AccessDeniedException exc) { + System.err.printf("ERROR: insufficient permissions to write to output directory [%s]%n", + this.outputDirectory); + return 1; + } + var launcher = new Launcher(); launcher.getEnvironment().setAutoImports(true); // add source code directories [specify as command line argument] @@ -72,10 +103,10 @@ public boolean matches(CtStatement element) { System.out.printf("%s%n%n", statement); } - // TODO write to specified file - ObjectMapper mapper = new ObjectMapper(); + var statementsFileName = Path.of(this.outputDirectory, "statements.json").toString(); + var mapper = new ObjectMapper(); mapper.enable(SerializationFeature.INDENT_OUTPUT); - try (var fileOutputStream = new FileOutputStream("statements.json")) { + try (var fileOutputStream = new FileOutputStream(statementsFileName)) { mapper.writeValue(fileOutputStream, statements); } From 7a3f395ae76393c5b60af808512c32eab429943f Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 16:36:00 -0500 Subject: [PATCH 02/11] Added Project class to Spoon analyser backend --- .../java/christimperley/kaskara/Main.java | 10 ++---- .../java/christimperley/kaskara/Project.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index 0384dfe..6e26fce 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -12,7 +12,6 @@ import java.util.List; import java.util.concurrent.Callable; import picocli.CommandLine; -import spoon.Launcher; import spoon.reflect.code.CtStatement; import spoon.reflect.visitor.filter.AbstractFilter; @@ -61,14 +60,11 @@ public Integer call() throws IOException { return 1; } - var launcher = new Launcher(); - launcher.getEnvironment().setAutoImports(true); - // add source code directories [specify as command line argument] - launcher.addInputResource(this.directory); - var model = launcher.buildModel(); + // construct a description of the project + var project = Project.build(this.directory); // find all statements in the program - var elements = model.getElements(new AbstractFilter() { + var elements = project.getModel().getElements(new AbstractFilter() { @Override public boolean matches(CtStatement element) { // must be a top-level statement within a block diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java new file mode 100644 index 0000000..70fc57a --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java @@ -0,0 +1,32 @@ +package christimperley.kaskara; + +import spoon.Launcher; +import spoon.reflect.CtModel; + +/** + * Maintains information about the program under analysis that is used by various classes. + */ +public class Project { + private final CtModel model; + + /** + * Constructs a description of a project whose source code is in a given directory. + * @param sourceDirectory The absolute path to the source code directory. + * @return A description of the project. + */ + public static Project build(String sourceDirectory) { + var launcher = new Launcher(); + launcher.getEnvironment().setAutoImports(true); + launcher.addInputResource(sourceDirectory); + var model = launcher.buildModel(); + return new Project(model); + } + + public Project(CtModel model) { + this.model = model; + } + + public final CtModel getModel() { + return this.model; + } +} From 575228533bfb46345b3156dd17509a8428169b2e Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 16:53:13 -0500 Subject: [PATCH 03/11] added separate StatementFinder class to Spoon backend --- .../java/christimperley/kaskara/Main.java | 67 ++++++------------- .../java/christimperley/kaskara/Project.java | 2 +- .../kaskara/StatementFinder.java | 62 +++++++++++++++++ 3 files changed, 85 insertions(+), 46 deletions(-) create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/StatementFinder.java diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index 6e26fce..e08187d 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -27,6 +27,8 @@ public class Main implements Callable { description = "The directory to which results should be written.") private String outputDirectory; + private ObjectMapper mapper; + /** * Provides an entrypoint to the Kaskara Java analysis tool. * @@ -50,6 +52,21 @@ private void prepareOutputDirectory() throws IOException { System.out.printf("Output will be written to: %s%n", this.outputDirectory); } + /** + * Finds all statements within the project and writes a summary of those statements + * to disk. + * @throws IOException If an error occurs during the write to disk. + */ + private void findStatements(Project project) throws IOException { + System.out.println("Finding all statements in project"); + var statementsFileName = Path.of(this.outputDirectory, "statements.json").toString(); + var statements = StatementFinder.forProject(project).find(); + try (var fileOutputStream = new FileOutputStream(statementsFileName)) { + this.mapper.writeValue(fileOutputStream, statements); + } + System.out.printf("Wrote summary of statements to disk [%s]%n", statementsFileName); + } + @Override public Integer call() throws IOException { try { @@ -60,52 +77,12 @@ public Integer call() throws IOException { return 1; } - // construct a description of the project - var project = Project.build(this.directory); - - // find all statements in the program - var elements = project.getModel().getElements(new AbstractFilter() { - @Override - public boolean matches(CtStatement element) { - // must be a top-level statement within a block - if (!(element.getParent() instanceof spoon.support.reflect.code.CtBlockImpl)) { - return false; - } - - // ignore blocks - if (element instanceof spoon.support.reflect.code.CtBlockImpl) { - return false; - } - - // ignore comments - if (element instanceof spoon.support.reflect.code.CtCommentImpl) { - return false; - } - - // ignore class implementations - if (element instanceof spoon.support.reflect.declaration.CtClassImpl) { - return false; - } - - // statement must appear in file - return element.getPosition().isValidPosition(); - } - }); - - List statements = new ArrayList<>(); - for (var element : elements) { - var statement = Statement.forSpoonStatement(element); - statements.add(statement); - System.out.printf("%s%n%n", statement); - } - - var statementsFileName = Path.of(this.outputDirectory, "statements.json").toString(); - var mapper = new ObjectMapper(); - mapper.enable(SerializationFeature.INDENT_OUTPUT); - try (var fileOutputStream = new FileOutputStream(statementsFileName)) { - mapper.writeValue(fileOutputStream, statements); - } + // prepare the JSON output formatter + this.mapper = new ObjectMapper(); + this.mapper.enable(SerializationFeature.INDENT_OUTPUT); + var project = Project.build(this.directory); + this.findStatements(project); return 0; } } diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java index 70fc57a..360ab2b 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Project.java @@ -22,7 +22,7 @@ public static Project build(String sourceDirectory) { return new Project(model); } - public Project(CtModel model) { + protected Project(CtModel model) { this.model = model; } diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/StatementFinder.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/StatementFinder.java new file mode 100644 index 0000000..262b64e --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/StatementFinder.java @@ -0,0 +1,62 @@ +package christimperley.kaskara; + +import java.util.ArrayList; +import java.util.List; +import spoon.reflect.code.CtStatement; +import spoon.reflect.visitor.filter.AbstractFilter; + +/** + * Provides an interface for finding all statements within a given project. + */ +public class StatementFinder { + private final Project project; + + public static StatementFinder forProject(Project project) { + return new StatementFinder(project); + } + + protected StatementFinder(Project project) { + this.project = project; + } + + /** + * Finds all statements within the associated project. + * @return A list of all statements within the associated project. + */ + public List find() { + var elements = this.project.getModel().getElements(new AbstractFilter() { + @Override + public boolean matches(CtStatement element) { + // must be a top-level statement within a block + if (!(element.getParent() instanceof spoon.support.reflect.code.CtBlockImpl)) { + return false; + } + + // ignore blocks + if (element instanceof spoon.support.reflect.code.CtBlockImpl) { + return false; + } + + // ignore comments + if (element instanceof spoon.support.reflect.code.CtCommentImpl) { + return false; + } + + // ignore class implementations + if (element instanceof spoon.support.reflect.declaration.CtClassImpl) { + return false; + } + + // statement must appear in file + return element.getPosition().isValidPosition(); + } + }); + + List statements = new ArrayList<>(); + for (var element : elements) { + var statement = Statement.forSpoonStatement(element); + statements.add(statement); + } + return statements; + } +} From 14864453550579542db96be66c032fe141d29a16 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 16:53:49 -0500 Subject: [PATCH 04/11] removed dead imports from spoon backend --- .../backend/src/main/java/christimperley/kaskara/Main.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index e08187d..e8d42d4 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -8,12 +8,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.Callable; import picocli.CommandLine; -import spoon.reflect.code.CtStatement; -import spoon.reflect.visitor.filter.AbstractFilter; @CommandLine.Command(name = "kaskara", mixinStandardHelpOptions = true) From ab3d5c499335ac95357595aae96c476a05922f66 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 17:06:19 -0500 Subject: [PATCH 05/11] added canonical property placeholder to Statement description for Spoon backend --- .../src/main/java/christimperley/kaskara/Statement.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java index b1f2f63..cf68765 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java @@ -42,6 +42,15 @@ public String getKind() { return this.kind.getName(); } + /** + * Returns the canonical form of the source for the statement. + * @return canonicalised source code + */ + @JsonGetter("canonical") + public String getCanonicalSource() { + return this.source; + } + @JsonGetter("source") public String getSource() { return this.source; From d9884540baff335eab8bbfa772035997f47d9d6a Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Tue, 4 Feb 2020 21:37:25 -0500 Subject: [PATCH 06/11] Implemented container construction for Spoon analyser --- lib/kaskara/spoon/analyser.py | 37 +++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/kaskara/spoon/analyser.py b/lib/kaskara/spoon/analyser.py index 569abea..93464fa 100644 --- a/lib/kaskara/spoon/analyser.py +++ b/lib/kaskara/spoon/analyser.py @@ -3,6 +3,10 @@ from typing import Iterator import contextlib +import json +import os +import shlex +import subprocess from dockerblade import DockerDaemon as DockerBladeDockerDaemon from loguru import logger @@ -22,15 +26,29 @@ class SpoonAnalyser(Analyser): @contextlib.contextmanager def _container(self, project: Project) -> Iterator[ProjectContainer]: """Provisions an ephemeral container for a given project.""" - create = self._dockerblade.client.containers.create launch = self._dockerblade.client.containers.run with contextlib.ExitStack() as stack: - docker_project = create(project.image) - stack.callback(docker_project.remove, force=True) + # create a temporary volume from the project image + volume_name = 'kaskaraspoon' + cmd_create_volume = (f'docker run --rm -v {volume_name}:' + f'{shlex.quote(project.directory)} ' + f'{project.image} /bin/true') + cmd_kill_volume = f'docker volume rm {volume_name}' + logger.debug(f'created temporary volume [{volume_name}] ' + f'from project image [{project.image}] ' + f'via command: {cmd_create_volume}') + subprocess.check_output(cmd_create_volume, shell=True) + stack.callback(subprocess.call, cmd_kill_volume, + shell=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stdin=subprocess.DEVNULL) docker_analyser = launch(SPOON_IMAGE_NAME, '/bin/sh', stdin_open=True, - volumes_from=[docker_project.id], + volumes={volume_name: { + 'bind': '/workspace', + 'mode': 'ro'}}, detach=True) stack.callback(docker_analyser.remove, force=True) @@ -43,4 +61,15 @@ def analyse(self, project: Project) -> Analysis: return self._analyse_container(container) def _analyse_container(self, container: ProjectContainer) -> Analysis: + dir_source = '/workspace' + dir_output_container = '/output' + command = f'kaskara {dir_source} -o {dir_output_container}' + output = container.shell.check_output(command) + + # TODO parse statements file + filename_statements_container = os.path.join(dir_output_container, + 'statements.json') + statements_dict = \ + json.loads(container.files.read(filename_statements_container)) + print(statements_dict) raise NotImplementedError From c7ba433a4e073278a07fbc962976fd1ddbc2d8da Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Wed, 5 Feb 2020 09:14:48 -0500 Subject: [PATCH 07/11] Implemented statement loading for Spoon analyser --- lib/kaskara/spoon/analyser.py | 19 ++++++++++-- lib/kaskara/spoon/analysis.py | 29 +++++++++++++++++++ .../christimperley/kaskara/Statement.java | 6 ++-- lib/kaskara/statements.py | 4 +++ 4 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 lib/kaskara/spoon/analysis.py diff --git a/lib/kaskara/spoon/analyser.py b/lib/kaskara/spoon/analyser.py index 93464fa..93fbb5f 100644 --- a/lib/kaskara/spoon/analyser.py +++ b/lib/kaskara/spoon/analyser.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- __all__ = ('SpoonAnalyser',) -from typing import Iterator +from typing import Any, Iterator, Mapping, Sequence import contextlib import json import os @@ -12,11 +12,13 @@ from loguru import logger import attr +from .analysis import SpoonStatement from .post_install import IMAGE_NAME as SPOON_IMAGE_NAME from ..analyser import Analyser from ..analysis import Analysis from ..container import ProjectContainer from ..project import Project +from ..statements import ProgramStatements @attr.s @@ -66,10 +68,21 @@ def _analyse_container(self, container: ProjectContainer) -> Analysis: command = f'kaskara {dir_source} -o {dir_output_container}' output = container.shell.check_output(command) - # TODO parse statements file + # load statements filename_statements_container = os.path.join(dir_output_container, 'statements.json') statements_dict = \ json.loads(container.files.read(filename_statements_container)) - print(statements_dict) + statements = self._load_statements_from_dict(statements_dict) + raise NotImplementedError + + def _load_statements_from_dict(self, + dict_: Sequence[Mapping[str, Any]] + ) -> ProgramStatements: + """Loads the statement database from a given dictionary.""" + logger.debug('parsing statements database') + statements = \ + ProgramStatements([SpoonStatement.from_dict(d) for d in dict_]) + logger.debug(f'parsed {len(statements)} statements') + return statements diff --git a/lib/kaskara/spoon/analysis.py b/lib/kaskara/spoon/analysis.py new file mode 100644 index 0000000..4f4e2b7 --- /dev/null +++ b/lib/kaskara/spoon/analysis.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +__all__ = ('SpoonStatement',) + +from typing import Any, FrozenSet, Mapping, Optional + +import attr + +from ..core import FileLocationRange +from ..statements import Statement + + +@attr.s(frozen=True, auto_attribs=True, slots=True) +class SpoonStatement(Statement): + kind: str + content: str + canonical: str + location: FileLocationRange + + @staticmethod + def from_dict(dict_: Mapping[str, Any]) -> 'SpoonStatement': + kind: str = dict_['kind'] + content: str = dict_['source'] + canonical: str = dict_['canonical'] + location = FileLocationRange.from_string(dict_['location']) + return SpoonStatement(kind, content, canonical, location) + + @property + def visible(self) -> Optional[FrozenSet[str]]: + return None diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java index cf68765..f09a615 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java @@ -57,10 +57,10 @@ public String getSource() { } /** - * Returns the position of statement as a string. - * @return A string encoding of the position of the statement. + * Returns the location of statement as a string. + * @return A string encoding of the location of the statement. */ - @JsonGetter("position") + @JsonGetter("location") public String getPositionAsString() { var filename = this.position.getFile(); var startLine = this.position.getLine(); diff --git a/lib/kaskara/statements.py b/lib/kaskara/statements.py index 943e367..6b45994 100644 --- a/lib/kaskara/statements.py +++ b/lib/kaskara/statements.py @@ -66,6 +66,10 @@ def __init__(self, statements: Iterable[Statement]) -> None: in self.__file_to_statements.items()) logger.debug(f'indexed statements by file:\n{summary}') + def __len__(self) -> int: + """Returns the number of statements in this collection.""" + return len(self.__statements) + def __iter__(self) -> Iterator[Statement]: yield from self.__statements From d633037e66a1264e28e3a07e1d613ba2a6909b4c Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Wed, 5 Feb 2020 10:02:25 -0500 Subject: [PATCH 08/11] Added function discovery prototype to Spoon backend --- .../java/christimperley/kaskara/Function.java | 43 +++++++++++++++++++ .../kaskara/FunctionFinder.java | 42 ++++++++++++++++++ .../java/christimperley/kaskara/Main.java | 22 ++++++++-- .../kaskara/SourcePositionSerializer.java | 16 +++++++ .../christimperley/kaskara/Statement.java | 29 +++---------- 5 files changed, 125 insertions(+), 27 deletions(-) create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/SourcePositionSerializer.java diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java new file mode 100644 index 0000000..6728ea4 --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java @@ -0,0 +1,43 @@ +package christimperley.kaskara; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import spoon.reflect.cu.SourcePosition; +import spoon.reflect.declaration.CtMethod; + +/** + * Describes a function within a given project. + */ +public class Function { + @JsonProperty("name") + private final String name; + @JsonSerialize(converter = SourcePositionSerializer.class) + private final SourcePosition location; + @JsonSerialize(converter = SourcePositionSerializer.class) + private final SourcePosition bodyLocation; + + /** + * Constructs a function description for a given Clang AST method element. + * @param element The AST element for the method. + * @return A description of the given AST element. + */ + public static Function forSpoonMethod(CtMethod element) { + var name = element.getSimpleName(); + var location = element.getPosition(); + var body = element.getBody(); + var bodyLocation = body.getPosition(); + return new Function(name, location, bodyLocation); + } + + /** + * Constructs a function description. + * @param name The name of the function. + * @param location The location of the function definition. + * @param bodyLocation The location of the body of the function definition. + */ + public Function(String name, SourcePosition location, SourcePosition bodyLocation) { + this.name = name; + this.location = location; + this.bodyLocation = bodyLocation; + } +} diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java new file mode 100644 index 0000000..1bd520d --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java @@ -0,0 +1,42 @@ +package christimperley.kaskara; + +import java.util.ArrayList; +import java.util.List; +import spoon.reflect.declaration.CtMethod; +import spoon.reflect.visitor.filter.AbstractFilter; + +/** + * Provides an interface for finding all function definitions within a given project. + */ +public class FunctionFinder { + private final Project project; + + public static FunctionFinder forProject(Project project) { + return new FunctionFinder(project); + } + + protected FunctionFinder(Project project) { + this.project = project; + } + + /** + * Finds all function declarations within the associated project. + * @return A list of all functions within the associated project. + */ + public List find() { + var elements = this.project.getModel().getElements(new AbstractFilter() { + @Override + public boolean matches(CtMethod element) { + // function must appear in file + return element.getPosition().isValidPosition(); + } + }); + + List functions = new ArrayList<>(); + for (var element : elements) { + var function = Function.forSpoonMethod(element); + functions.add(function); + } + return functions; + } +} diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index e8d42d4..b44b53c 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -55,12 +55,27 @@ private void prepareOutputDirectory() throws IOException { */ private void findStatements(Project project) throws IOException { System.out.println("Finding all statements in project"); - var statementsFileName = Path.of(this.outputDirectory, "statements.json").toString(); + var filename = Path.of(this.outputDirectory, "statements.json").toString(); var statements = StatementFinder.forProject(project).find(); - try (var fileOutputStream = new FileOutputStream(statementsFileName)) { + try (var fileOutputStream = new FileOutputStream(filename)) { this.mapper.writeValue(fileOutputStream, statements); } - System.out.printf("Wrote summary of statements to disk [%s]%n", statementsFileName); + System.out.printf("Wrote summary of statements to disk [%s]%n", filename); + } + + /** + * Finds all functions within the project and writes a summary of those functions + * to disk. + * @throws IOException If an error occurs during the write to disk. + */ + private void findFunctions(Project project) throws IOException { + System.out.println("Finding all functions in project"); + var filename = Path.of(this.outputDirectory, "functions.json").toString(); + var functions = FunctionFinder.forProject(project).find(); + try (var fileOutputStream = new FileOutputStream(filename)) { + this.mapper.writeValue(fileOutputStream, functions); + } + System.out.printf("Wrote summary of functions to disk [%s]%n", filename); } @Override @@ -79,6 +94,7 @@ public Integer call() throws IOException { var project = Project.build(this.directory); this.findStatements(project); + this.findFunctions(project); return 0; } } diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/SourcePositionSerializer.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/SourcePositionSerializer.java new file mode 100644 index 0000000..a3b3bee --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/SourcePositionSerializer.java @@ -0,0 +1,16 @@ +package christimperley.kaskara; + +import com.fasterxml.jackson.databind.util.StdConverter; +import spoon.reflect.cu.SourcePosition; + +public class SourcePositionSerializer extends StdConverter { + @Override + public String convert(SourcePosition value) { + return String.format("%s@%d:%d::%d:%d", + value.getFile().getAbsolutePath(), + value.getLine(), + value.getColumn(), + value.getEndLine(), + value.getEndColumn()); + } +} diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java index f09a615..078affe 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java @@ -1,6 +1,8 @@ package christimperley.kaskara; import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import spoon.reflect.code.CtStatement; import spoon.reflect.cu.SourcePosition; @@ -10,7 +12,10 @@ */ public final class Statement { private final Class kind; + @JsonProperty("source") private final String source; + + @JsonSerialize(converter = SourcePositionSerializer.class) private final SourcePosition position; /** @@ -51,30 +56,6 @@ public String getCanonicalSource() { return this.source; } - @JsonGetter("source") - public String getSource() { - return this.source; - } - - /** - * Returns the location of statement as a string. - * @return A string encoding of the location of the statement. - */ - @JsonGetter("location") - public String getPositionAsString() { - var filename = this.position.getFile(); - var startLine = this.position.getLine(); - var startCol = this.position.getColumn(); - var endLine = this.position.getEndLine(); - var endCol = this.position.getEndColumn(); - return String.format("%s@%d:%d::%d:%d", - this.position.getFile().getAbsolutePath(), - this.position.getLine(), - this.position.getColumn(), - this.position.getEndLine(), - this.position.getEndColumn()); - } - @Override public boolean equals(Object other) { if (!(other instanceof Statement)) { From 4ddb4a0c44eb305c0d7ed18579644159ab09e799 Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Wed, 5 Feb 2020 16:31:15 -0500 Subject: [PATCH 09/11] Implemented frontend for Spoon function discovery --- lib/kaskara/spoon/analyser.py | 29 ++++++++++++++----- lib/kaskara/spoon/analysis.py | 17 +++++++++++ .../java/christimperley/kaskara/Function.java | 20 +++++++++++-- .../kaskara/FunctionFinder.java | 4 +++ .../christimperley/kaskara/Statement.java | 1 + 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/lib/kaskara/spoon/analyser.py b/lib/kaskara/spoon/analyser.py index 93fbb5f..c08b35c 100644 --- a/lib/kaskara/spoon/analyser.py +++ b/lib/kaskara/spoon/analyser.py @@ -12,11 +12,12 @@ from loguru import logger import attr -from .analysis import SpoonStatement +from .analysis import SpoonFunction, SpoonStatement from .post_install import IMAGE_NAME as SPOON_IMAGE_NAME from ..analyser import Analyser from ..analysis import Analysis from ..container import ProjectContainer +from ..functions import ProgramFunctions from ..project import Project from ..statements import ProgramStatements @@ -64,17 +65,19 @@ def analyse(self, project: Project) -> Analysis: def _analyse_container(self, container: ProjectContainer) -> Analysis: dir_source = '/workspace' - dir_output_container = '/output' - command = f'kaskara {dir_source} -o {dir_output_container}' - output = container.shell.check_output(command) + dir_output = '/output' + container.shell.check_output(f'kaskara {dir_source} -o {dir_output}') # load statements - filename_statements_container = os.path.join(dir_output_container, - 'statements.json') - statements_dict = \ - json.loads(container.files.read(filename_statements_container)) + filename_statements = os.path.join(dir_output, 'statements.json') + statements_dict = json.loads(container.files.read(filename_statements)) statements = self._load_statements_from_dict(statements_dict) + # load functions + filename_functions = os.path.join(dir_output, 'functions.json') + functions_dict = json.loads(container.files.read(filename_functions)) + functions = self._load_functions_from_dict(functions_dict) + raise NotImplementedError def _load_statements_from_dict(self, @@ -86,3 +89,13 @@ def _load_statements_from_dict(self, ProgramStatements([SpoonStatement.from_dict(d) for d in dict_]) logger.debug(f'parsed {len(statements)} statements') return statements + + def _load_functions_from_dict(self, + dict_: Sequence[Mapping[str, Any]] + ) -> ProgramFunctions: + """Loads the function database from a given dictionary.""" + logger.debug('parsing function database') + functions = \ + ProgramFunctions([SpoonFunction.from_dict(d) for d in dict_]) + logger.debug(f'parsed {len(functions)} functions') + return functions diff --git a/lib/kaskara/spoon/analysis.py b/lib/kaskara/spoon/analysis.py index 4f4e2b7..e9eb364 100644 --- a/lib/kaskara/spoon/analysis.py +++ b/lib/kaskara/spoon/analysis.py @@ -6,9 +6,26 @@ import attr from ..core import FileLocationRange +from ..functions import Function from ..statements import Statement +@attr.s(frozen=True, slots=True, auto_attribs=True) +class SpoonFunction(Function): + name: str + location: FileLocationRange + body_location: FileLocationRange + return_type: str + + @staticmethod + def from_dict(dict_: Mapping[str, Any]) -> 'SpoonFunction': + name: str = dict_['name'] + location = FileLocationRange.from_string(dict_['location']) + body_location = FileLocationRange.from_string(dict_['body']) + return_type = dict_['return-type'] + return SpoonFunction(name, location, body_location, return_type) + + @attr.s(frozen=True, auto_attribs=True, slots=True) class SpoonStatement(Statement): kind: str diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java index 6728ea4..0703e0f 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Function.java @@ -1,9 +1,11 @@ package christimperley.kaskara; +import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import spoon.reflect.cu.SourcePosition; import spoon.reflect.declaration.CtMethod; +import spoon.reflect.reference.CtTypeReference; /** * Describes a function within a given project. @@ -12,9 +14,12 @@ public class Function { @JsonProperty("name") private final String name; @JsonSerialize(converter = SourcePositionSerializer.class) + @JsonProperty("location") private final SourcePosition location; @JsonSerialize(converter = SourcePositionSerializer.class) + @JsonProperty("body") private final SourcePosition bodyLocation; + private final CtTypeReference returnType; /** * Constructs a function description for a given Clang AST method element. @@ -26,7 +31,8 @@ public static Function forSpoonMethod(CtMethod element) { var location = element.getPosition(); var body = element.getBody(); var bodyLocation = body.getPosition(); - return new Function(name, location, bodyLocation); + var returnType = element.getType(); + return new Function(name, location, bodyLocation, returnType); } /** @@ -34,10 +40,20 @@ public static Function forSpoonMethod(CtMethod element) { * @param name The name of the function. * @param location The location of the function definition. * @param bodyLocation The location of the body of the function definition. + * @param returnType The return type of the function. */ - public Function(String name, SourcePosition location, SourcePosition bodyLocation) { + public Function(String name, + SourcePosition location, + SourcePosition bodyLocation, + CtTypeReference returnType) { this.name = name; this.location = location; this.bodyLocation = bodyLocation; + this.returnType = returnType; + } + + @JsonGetter("return-type") + public String getReturnType() { + return this.returnType.getQualifiedName(); } } diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java index 1bd520d..acc4d6d 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/FunctionFinder.java @@ -27,6 +27,10 @@ public List find() { var elements = this.project.getModel().getElements(new AbstractFilter() { @Override public boolean matches(CtMethod element) { + // function must have body + if (element.getBody() == null) { + return false; + } // function must appear in file return element.getPosition().isValidPosition(); } diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java index 078affe..692a6cf 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Statement.java @@ -16,6 +16,7 @@ public final class Statement { private final String source; @JsonSerialize(converter = SourcePositionSerializer.class) + @JsonProperty("location") private final SourcePosition position; /** From 36479114082fabd1b9912aa7da333f36512796fb Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Wed, 5 Feb 2020 17:17:33 -0500 Subject: [PATCH 10/11] Added loop finder to Spoon backend --- .../java/christimperley/kaskara/Loop.java | 29 ++++++++++++ .../christimperley/kaskara/LoopFinder.java | 45 +++++++++++++++++++ .../java/christimperley/kaskara/Main.java | 15 +++++++ 3 files changed, 89 insertions(+) create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Loop.java create mode 100644 lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/LoopFinder.java diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Loop.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Loop.java new file mode 100644 index 0000000..e5da196 --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Loop.java @@ -0,0 +1,29 @@ +package christimperley.kaskara; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import spoon.reflect.code.CtLoop; +import spoon.reflect.cu.SourcePosition; + +/** + * Describes a control-flow loop within a given project. + */ +public class Loop { + @JsonSerialize(converter = SourcePositionSerializer.class) + @JsonProperty("body") + private final SourcePosition bodyLocation; + + /** + * Constructs a description for a given Clang AST loop element. + * @param element The AST element for the loop. + * @return A description of the given AST element. + */ + public static Loop forSpoonLoop(CtLoop element) { + var bodyLocation = element.getBody().getPosition(); + return new Loop(bodyLocation); + } + + protected Loop(SourcePosition bodyLocation) { + this.bodyLocation = bodyLocation; + } +} diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/LoopFinder.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/LoopFinder.java new file mode 100644 index 0000000..00f179f --- /dev/null +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/LoopFinder.java @@ -0,0 +1,45 @@ +package christimperley.kaskara; + +import java.util.ArrayList; +import java.util.List; +import spoon.reflect.code.CtLoop; +import spoon.reflect.visitor.filter.AbstractFilter; + +/** + * Provides an interface for finding all loops within a given project. + */ +public class LoopFinder { + private final Project project; + + public static LoopFinder forProject(Project project) { + return new LoopFinder(project); + } + + protected LoopFinder(Project project) { + this.project = project; + } + + /** + * Finds all loops within the associated project. + * @return A list of all loops within the associated project. + */ + public List find() { + var elements = this.project.getModel().getElements(new AbstractFilter() { + @Override + public boolean matches(CtLoop element) { + // loop must have body + if (element.getBody() == null) { + return false; + } + // loop must appear in file + return element.getPosition().isValidPosition(); + } + }); + + List loops = new ArrayList<>(); + for (var element : elements) { + loops.add(Loop.forSpoonLoop(element)); + } + return loops; + } +} diff --git a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java index b44b53c..0eb94ea 100644 --- a/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java +++ b/lib/kaskara/spoon/backend/src/main/java/christimperley/kaskara/Main.java @@ -78,6 +78,20 @@ private void findFunctions(Project project) throws IOException { System.out.printf("Wrote summary of functions to disk [%s]%n", filename); } + /** + * Finds all loops within the project and writes a summary of those functions to disk. + * @throws IOException If an error occurs during the write to disk. + */ + private void findLoops(Project project) throws IOException { + System.out.println("Finding all loops in project"); + var filename = Path.of(this.outputDirectory, "loops.json").toString(); + var loops = LoopFinder.forProject(project).find(); + try (var fileOutputStream = new FileOutputStream(filename)) { + this.mapper.writeValue(fileOutputStream, loops); + } + System.out.printf("Wrote summary of loops to disk [%s]%n", filename); + } + @Override public Integer call() throws IOException { try { @@ -95,6 +109,7 @@ public Integer call() throws IOException { var project = Project.build(this.directory); this.findStatements(project); this.findFunctions(project); + this.findLoops(project); return 0; } } From 8ddfb991e020e8a81d21b9787852e6f64a6051ab Mon Sep 17 00:00:00 2001 From: ChrisTimperley Date: Wed, 5 Feb 2020 17:37:36 -0500 Subject: [PATCH 11/11] Added loop discovery to Spoon analyser --- lib/kaskara/spoon/analyser.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/kaskara/spoon/analyser.py b/lib/kaskara/spoon/analyser.py index c08b35c..c79310a 100644 --- a/lib/kaskara/spoon/analyser.py +++ b/lib/kaskara/spoon/analyser.py @@ -17,7 +17,9 @@ from ..analyser import Analyser from ..analysis import Analysis from ..container import ProjectContainer +from ..core import FileLocationRange from ..functions import ProgramFunctions +from ..loops import ProgramLoops from ..project import Project from ..statements import ProgramStatements @@ -78,7 +80,19 @@ def _analyse_container(self, container: ProjectContainer) -> Analysis: functions_dict = json.loads(container.files.read(filename_functions)) functions = self._load_functions_from_dict(functions_dict) - raise NotImplementedError + # load loops + filename_loops = os.path.join(dir_output, 'loops.json') + loops_dict = json.loads(container.files.read(filename_loops)) + loops = self._load_loops_from_dict(loops_dict) + + # find insertion points + insertions = statements.insertions() + + return Analysis(project=container.project, + loops=loops, + functions=functions, + statements=statements, + insertions=insertions) def _load_statements_from_dict(self, dict_: Sequence[Mapping[str, Any]] @@ -99,3 +113,16 @@ def _load_functions_from_dict(self, ProgramFunctions([SpoonFunction.from_dict(d) for d in dict_]) logger.debug(f'parsed {len(functions)} functions') return functions + + def _load_loops_from_dict(self, + dict_: Sequence[Mapping[str, Any]] + ) -> ProgramFunctions: + """Loads the loops database from a given dictionary.""" + logger.debug('parsing loop database') + loop_bodies: List[FileLocationRange] = [] + for loop_info in dict_: + loc = FileLocationRange.from_string(loop_info['body']) + loop_bodies.append(loc) + loops = ProgramLoops.from_body_location_ranges(loop_bodies) + logger.debug(f'parsed loops') + return loops