Add NestedFileSystemOperationNodes, canonical wire format and round-t…

…ripping.

Since NestedFileSystemOperationNodes are identified by a fingerprint of
their serialized representation uses a custom, canonical wire format. As
per https://protobuf.dev/programming-guides/serialization-not-canonical/,
protos are not canonical.

Performs some renamings for consistency.
* In some places renames Directory to Listing. This emphasizes that it's
the listing that matters for invalidation.
* For consistency, some places that were DirectoryListing are also
renamed as Listing.
* GetDependenciesResult is renamed to GetFileDependenciesResult for
consistency.

PiperOrigin-RevId: 688712493
Change-Id: I0456a3d7ab0a8f4077d057d9ded3f1061fd85d11

src/main/java/com/google/devtools/build/lib/skyframe/BUILD

@@ -1506,13 +1506,16 @@ java_library(
         "DirectoryListingKey.java",
         "FileKey.java",
         "FileSystemOperationNode.java",
+        "NestedFileSystemOperationNodes.java",
     ],
     deps = [
         ":sky_functions",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization:visible-for-serialization",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:error_prone_annotations",
+        "//third_party:jsr305",
     ],
 )
 

src/main/java/com/google/devtools/build/lib/skyframe/FileSystemOperationNode.java

@@ -13,5 +13,10 @@
 // limitations under the License.
 package com.google.devtools.build.lib.skyframe;
 
-/** A {@link FileKey} or {@link DirectoryListingKey}. */
-public sealed interface FileSystemOperationNode permits FileKey, DirectoryListingKey {}
+/**
+ * Represents either a single file system operation or a nested set thereof.
+ *
+ * <p>If the set is unary, it is represented without a wrapper.
+ */
+public sealed interface FileSystemOperationNode
+    permits FileKey, DirectoryListingKey, NestedFileSystemOperationNodes {}

src/main/java/com/google/devtools/build/lib/skyframe/NestedFileSystemOperationNodes.java

@@ -0,0 +1,88 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.util.ArrayList;
+import javax.annotation.Nullable;
+
+/** Represents multiple {@link FileSystemOperationNode}s with nestable composition. */
+public final class NestedFileSystemOperationNodes implements FileSystemOperationNode {
+  private final FileSystemOperationNode[] nodes;
+
+  /**
+   * Opaque storage for use by serialization.
+   *
+   * <p>{@link FileSystemOperationNode}, {@link FileKey} and {@link DirectoryListingKey} are
+   * mutually dependent via {@link FileSystemOperationNode}. This type is opaque to avoid forcing
+   * {@link FileKey} and {@link DirectoryListingKey} to depend on serialization implementation code.
+   *
+   * <p>The serialization implementation initializes this field with double-checked locking so it is
+   * marked volatile.
+   */
+  private volatile Object serializationScratch;
+
+  public static FileSystemOperationNodeBuilder builder(FileSystemOperationNode node) {
+    return new FileSystemOperationNodeBuilder(node);
+  }
+
+  private NestedFileSystemOperationNodes(FileSystemOperationNode[] nodes) {
+    this.nodes = nodes;
+  }
+
+  public int count() {
+    return nodes.length;
+  }
+
+  public FileSystemOperationNode get(int index) {
+    return nodes[index];
+  }
+
+  @Nullable
+  public Object getSerializationScratch() {
+    return serializationScratch;
+  }
+
+  public void setSerializationScratch(Object value) {
+    this.serializationScratch = value;
+  }
+
+  /**
+   * Effectively, a builder for {@link NestedFileSystemOperationNodes}, but formally a builder for
+   * {@link FileSystemOperationNode}.
+   *
+   * <p>When there is only one node, this builder returns the node directly instead of creating a
+   * useless wrapper.
+   */
+  public static class FileSystemOperationNodeBuilder {
+    private final ArrayList<FileSystemOperationNode> nodes = new ArrayList<>();
+
+    private FileSystemOperationNodeBuilder(FileSystemOperationNode node) {
+      nodes.add(node);
+    }
+
+    @CanIgnoreReturnValue
+    public FileSystemOperationNodeBuilder add(FileSystemOperationNode node) {
+      nodes.add(node);
+      return this;
+    }
+
+    public FileSystemOperationNode build() {
+      if (nodes.size() > 1) {
+        return new NestedFileSystemOperationNodes(nodes.toArray(FileSystemOperationNode[]::new));
+      }
+      return nodes.get(0);
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/skyframe/serialization/PackedFingerprint.java

@@ -36,7 +36,8 @@
  * @param lo the lower 64-bits of the fingerprint
  * @param hi the upper 64-bits of the fingerprint
  */
-public record PackedFingerprint(long lo, long hi) implements KeyBytesProvider {
+public record PackedFingerprint(long lo, long hi)
+    implements KeyBytesProvider, Comparable<PackedFingerprint> {
   @VisibleForTesting static final int BYTES = 16;
 
   /**
@@ -108,6 +109,15 @@ public int hashCode() {
     return (int) lo;
   }
 
+  @Override
+  public int compareTo(PackedFingerprint o) {
+    int result = Long.compare(hi, o.hi);
+    if (result == 0) {
+      return Long.compare(lo, o.lo);
+    }
+    return result;
+  }
+
   /** Changes all 0 bytes of {@code input} into 1. */
   @VisibleForTesting
   static long offsetZeros(long input) {

src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD

@@ -75,20 +75,23 @@ java_library(
         "//src/main/protobuf:file_invalidation_data_java_proto",
         "//third_party:guava",
         "//third_party:jsr305",
+        "@protobuf//:protobuf_java",
     ],
 )
 
 java_library(
     name = "file_dependency_deserializer",
     srcs = [
-        "DirectoryListingDependencies.java",
         "FileDependencies.java",
         "FileDependencyDeserializer.java",
         "FileSystemDependencies.java",
+        "ListingDependencies.java",
+        "NestedDependencies.java",
     ],
     deps = [
         ":file_dependency_key_support",
         ":settable_future_with_ownership",
+        "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
         "//src/main/java/com/google/devtools/build/lib/vfs:ospathpolicy",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",

src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/FileDependencies.java

@@ -32,7 +32,7 @@
  * value is invalidated.
  */
 abstract sealed class FileDependencies
-    implements FileSystemDependencies, FileDependencyDeserializer.GetDependenciesResult
+    implements FileSystemDependencies, FileDependencyDeserializer.GetFileDependenciesResult
     permits FileDependencies.SingleResolvedPath,
         FileDependencies.SingleResolvedPathAndDependency,
         FileDependencies.MultiplePaths {