Handle .srcjar inputs in Bazel aspect (#754)

* Unpack .srcjars in Bazel aspect and add them to scip config

* Use --verbose_failures when invoking scip-java aspect

* Scip buildtool can now handle folders of source files

* Improve compatibility with different Bazel versions

* remove redundant conversion

* Add a srcjar example/test

examples/bazel-example/src/main/java/srcjar_example/BUILD

@@ -0,0 +1,13 @@
+genrule(
+    name = "generated-srcjar",
+    outs = ["sources.srcjar"],
+    cmd = "echo 'package com.testing; public class Bar {};' > Bar.java && jar cf $(@) Bar.java",
+)
+
+java_library(
+    name = "testing",
+    srcs = [
+        "Foo.java",
+        ":generated-srcjar",
+    ],
+)

examples/bazel-example/src/main/java/srcjar_example/Foo.java

@@ -0,0 +1,8 @@
+// testing/Foo.java
+package com.testing;
+
+public class Foo {
+    public Bar foo(Bar value) {
+        return value;
+    }
+}

scip-java/src/main/resources/scip-java/scip_java.bzl

@@ -57,10 +57,36 @@ def _scip_java(target, ctx):
     annotations = info.annotation_processing
 
     source_files = []
+    source_jars = []
     for src in ctx.rule.files.srcs:
-        source_files.append(src.path)
+        if src.path.endswith(".java"):
+            source_files.append(src.path)
+        elif src.path.endswith(".srcjar"):
+            source_jars.append(src)
+
     if len(source_files) == 0:
         return None
+
+    output_dir = []
+
+    for source_jar in source_jars:
+        dir = ctx.actions.declare_directory("extracted_srcjar/" + source_jar.short_path)
+        output_dir.append(dir)
+
+        ctx.actions.run_shell(
+            inputs = javac_action.inputs,
+            outputs = [dir],
+            mnemonic = "ExtractSourceJars",
+            command = """
+                unzip {input_file} -d {output_dir}
+            """.format(
+                output_dir = dir.path,
+                input_file = source_jar.path,
+            ),
+            progress_message = "Extracting source jar {jar}".format(jar = source_jar.path),
+        )
+
+        source_files.append(dir.path)
 
     classpath = [j.path for j in compilation.compilation_classpath.to_list()]
     bootclasspath = [j.path for j in compilation.boot_classpath]
@@ -73,11 +99,23 @@ def _scip_java(target, ctx):
 
     launcher_javac_flags = []
     compiler_javac_flags = []
-    for value in compilation.javac_options:
-        if value.startswith("-J"):
-            launcher_javac_flags.append(value)
-        else:
-            compiler_javac_flags.append(value)
+
+    # In different versions of bazel javac options are either a nested set or a depset or a list...
+    javac_options = []
+    if hasattr(compilation, "javac_options_list"):
+        javac_options = compilation.javac_options_list
+    else:
+        javac_options = compilation.javac_options
+
+    for value in javac_options:
+        # NOTE(Anton): for some bizarre reason I see empty string starting the list of 
+        # javac options - which then gets propagated into the JSON config, and ends up
+        # crashing the actual javac invokation.
+        if value != "":
+            if value.startswith("-J"):
+                launcher_javac_flags.append(value)
+            else:
+                compiler_javac_flags.append(value)
 
     build_config = struct(**{
         "javaHome": ctx.var["java_home"],
@@ -100,6 +138,7 @@ def _scip_java(target, ctx):
     )
 
     deps = [javac_action.inputs, annotations.processor_classpath]
+
     ctx.actions.run_shell(
         command = "\"{}\" index --no-cleanup --index-semanticdb.allow-empty-index --cwd \"{}\" --targetroot {} --scip-config \"{}\" --output \"{}\"".format(
             ctx.var["scip_java_binary"],
@@ -113,7 +152,7 @@ def _scip_java(target, ctx):
             "NO_PROGRESS_BAR": "true",
         },
         mnemonic = "ScipJavaIndex",
-        inputs = depset([build_config_path], transitive = deps),
+        inputs = depset([build_config_path] + output_dir, transitive = deps),
         outputs = [scip_output, targetroot],
     )
 

scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/BazelBuildTool.scala

@@ -71,7 +71,8 @@ class BazelBuildTool(index: IndexCommand) extends BuildTool("Bazel", index) {
         "--output_groups=scip",
         s"--define=sourceroot=${index.workingDirectory}",
         s"--define=java_home=$javaHome",
-        s"--define=scip_java_binary=$scipJavaBinary"
+        s"--define=scip_java_binary=$scipJavaBinary",
+        "--verbose_failures"
       ) ++ targetSpecs
 
     val buildExitCode = runBazelBuild(buildCommand)

scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala

@@ -683,15 +683,8 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
     Files.walkFileTree(targetroot, new DeleteVisitor)
   }
 
-  /** Recursively collects all Java files in the working directory */
-  private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = {
-    if (config.sourceFiles.nonEmpty) {
-      return config
-        .sourceFiles
-        .map(path => AbsolutePath.of(Paths.get(path), dir))
-        .filter(path => Files.isRegularFile(path))
-    }
-    val buf = ListBuffer.empty[Path]
+  private def collectAllSourceFiles(dir: Path) = {
+    val buf = List.newBuilder[Path]
     Files.walkFileTree(
       dir,
       new SimpleFileVisitor[Path] {
@@ -719,7 +712,26 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
         ): FileVisitResult = FileVisitResult.CONTINUE
       }
     )
-    buf.toList
+    buf.result()
+  }
+
+  /** Recursively collects all Java files in the working directory */
+  private def collectAllSourceFiles(config: Config, dir: Path): List[Path] = {
+    if (config.sourceFiles.nonEmpty) {
+      config
+        .sourceFiles
+        .flatMap { relativePath =>
+          val path = AbsolutePath.of(Paths.get(relativePath), dir)
+
+          if (Files.isRegularFile(path) && allPatterns.matches(path))
+            List(path)
+          else if (Files.isDirectory(path))
+            collectAllSourceFiles(path)
+          else
+            Nil
+        }
+    } else
+      collectAllSourceFiles(dir)
   }
 
   // HACK(olafurpg): I haven't figured out a reliable way to get annotation processor jars on the processorpath.