HDDS-12057. Implement ozone debug replicas verify command

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/FindMissingPadding.java

@@ -71,16 +71,16 @@
 /**
  * Find EC keys affected by missing padding blocks (HDDS-10681).
  */
-@CommandLine.Command(name = "find-missing-padding",
-    aliases = { "fmp" },
+@CommandLine.Command(name = "padding",
+    aliases = { "fmp", "find-missing-padding" },
     description = "List all keys with any missing padding, optionally limited to a volume/bucket/key URI.")
 @MetaInfServices(DebugSubcommand.class)
 public class FindMissingPadding extends Handler implements DebugSubcommand {
 
   @CommandLine.Mixin
   private ScmOption scmOption;
 
-  @CommandLine.Parameters(arity = "0..1",
+  @CommandLine.Parameters(arity = "1",
       description = Shell.OZONE_URI_DESCRIPTION)
   private String uri;
 

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ReadReplicas.java

@@ -23,15 +23,21 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.server.JsonUtils;
 import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.client.OzoneKey;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.ozone.common.OzoneChecksumException;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.shell.Handler;
 import org.apache.hadoop.ozone.shell.OzoneAddress;
-import org.apache.hadoop.ozone.shell.keys.KeyHandler;
+import org.apache.hadoop.ozone.shell.Shell;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import jakarta.annotation.Nonnull;
@@ -48,6 +54,7 @@
 import java.nio.file.StandardCopyOption;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
@@ -57,11 +64,12 @@
  * given key. It also generates a manifest file with information about the
  * downloaded replicas.
  */
-@CommandLine.Command(name = "read-replicas",
+@CommandLine.Command(name = "checksums",
     description = "Reads every replica for all the blocks associated with a " +
-        "given key.")
+        "given key.",
+    aliases = {"read-replicas"})
 @MetaInfServices(DebugSubcommand.class)
-public class ReadReplicas extends KeyHandler implements DebugSubcommand {
+public class ReadReplicas extends Handler implements DebugSubcommand {
 
   @CommandLine.Option(names = {"--outputDir", "-o", "--output-dir"},
       description = "Destination where the directory will be created" +
@@ -82,67 +90,20 @@ public class ReadReplicas extends KeyHandler implements DebugSubcommand {
   private static final String JSON_PROPERTY_REPLICA_UUID = "uuid";
   private static final String JSON_PROPERTY_REPLICA_EXCEPTION = "exception";
 
+  @CommandLine.Parameters(arity = "1",
+      description = Shell.OZONE_URI_DESCRIPTION)
+  private String uri;
+
+  @Override
+  protected OzoneAddress getAddress() throws OzoneClientException {
+    return new OzoneAddress(uri);
+  }
+
   @Override
   protected void execute(OzoneClient client, OzoneAddress address)
       throws IOException {
 
-    address.ensureKeyAddress();
-
-    boolean isChecksumVerifyEnabled
-        = getConf().getBoolean("ozone.client.verify.checksum", true);
-    OzoneConfiguration configuration = new OzoneConfiguration(getConf());
-    configuration.setBoolean("ozone.client.verify.checksum",
-        !isChecksumVerifyEnabled);
-
-    RpcClient newClient = new RpcClient(configuration, null);
-    try {
-      ClientProtocol noChecksumClient;
-      ClientProtocol checksumClient;
-      if (isChecksumVerifyEnabled) {
-        checksumClient = client.getObjectStore().getClientProxy();
-        noChecksumClient = newClient;
-      } else {
-        checksumClient = newClient;
-        noChecksumClient = client.getObjectStore().getClientProxy();
-      }
-
-      String volumeName = address.getVolumeName();
-      String bucketName = address.getBucketName();
-      String keyName = address.getKeyName();
-
-      File dir = createDirectory(volumeName, bucketName, keyName);
-
-      OzoneKeyDetails keyInfoDetails
-          = checksumClient.getKeyDetails(volumeName, bucketName, keyName);
-
-      Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicas =
-          checksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName);
-
-      Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>>
-          replicasWithoutChecksum = noChecksumClient
-          .getKeysEveryReplicas(volumeName, bucketName, keyName);
-
-      ObjectNode result = JsonUtils.createObjectNode(null);
-      result.put(JSON_PROPERTY_FILE_NAME,
-          volumeName + "/" + bucketName + "/" + keyName);
-      result.put(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize());
-
-      ArrayNode blocks = JsonUtils.createArrayNode();
-      downloadReplicasAndCreateManifest(keyName, replicas,
-          replicasWithoutChecksum, dir, blocks);
-      result.set(JSON_PROPERTY_FILE_BLOCKS, blocks);
-
-      String prettyJson = JsonUtils.toJsonStringWithDefaultPrettyPrinter(result);
-
-      String manifestFileName = keyName + "_manifest";
-      System.out.println("Writing manifest file : " + manifestFileName);
-      File manifestFile
-          = new File(dir, manifestFileName);
-      Files.write(manifestFile.toPath(),
-          prettyJson.getBytes(StandardCharsets.UTF_8));
-    } finally {
-      newClient.close();
-    }
+    findCandidateKeys(client, address);
   }
 
   private void downloadReplicasAndCreateManifest(
@@ -243,4 +204,91 @@ private File createDirectory(String volumeName, String bucketName,
     }
     return dir;
   }
+
+  private void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOException {
+    ObjectStore objectStore = ozoneClient.getObjectStore();
+    String volumeName = address.getVolumeName();
+    String bucketName = address.getBucketName();
+    String keyName = address.getKeyName();
+    if (!keyName.isEmpty()) {
+      processKey(ozoneClient, volumeName, bucketName, keyName);
+    } else if (!bucketName.isEmpty()) {
+      OzoneVolume volume = objectStore.getVolume(volumeName);
+      OzoneBucket bucket = volume.getBucket(bucketName);
+      checkBucket(bucket, ozoneClient);
+    } else if (!volumeName.isEmpty()) {
+      OzoneVolume volume = objectStore.getVolume(volumeName);
+      checkVolume(volume, ozoneClient);
+    } else {
+      for (Iterator<? extends OzoneVolume> it = objectStore.listVolumes(null); it.hasNext();) {
+        checkVolume(it.next(), ozoneClient);
+      }
+    }
+  }
+
+  private void checkVolume(OzoneVolume volume, OzoneClient ozoneClient) throws IOException {
+    for (Iterator<? extends OzoneBucket> it = volume.listBuckets(null); it.hasNext();) {
+      OzoneBucket bucket = it.next();
+      checkBucket(bucket, ozoneClient);
+    }
+  }
+
+  private void checkBucket(OzoneBucket bucket, OzoneClient ozoneClient) throws IOException {
+    String volumeName = bucket.getVolumeName();
+    String bucketName = bucket.getName();
+    for (Iterator<? extends OzoneKey> it = bucket.listKeys(null); it.hasNext();) {
+      OzoneKey key = it.next();
+//    TODO: Remove this check once HDDS-12094 is fixed
+      if (key.getName().endsWith("/")) {
+        continue;
+      }
+      processKey(ozoneClient, volumeName, bucketName, key.getName());
+    }
+  }
+
+  private void processKey(OzoneClient client, String volumeName, String bucketName, String keyName) throws IOException {
+    System.out.println("Processing key : " + volumeName + "/" + bucketName + "/" + keyName);
+    boolean isChecksumVerifyEnabled
+        = getConf().getBoolean("ozone.client.verify.checksum", true);
+    OzoneConfiguration configuration = new OzoneConfiguration(getConf());
+    configuration.setBoolean("ozone.client.verify.checksum",
+        !isChecksumVerifyEnabled);
+    RpcClient newClient = new RpcClient(configuration, null);
+
+    try {
+      ClientProtocol noChecksumClient;
+      ClientProtocol checksumClient;
+      if (isChecksumVerifyEnabled) {
+        checksumClient = client.getObjectStore().getClientProxy();
+        noChecksumClient = newClient;
+      } else {
+        checksumClient = newClient;
+        noChecksumClient = client.getObjectStore().getClientProxy();
+      }
+
+      File dir = createDirectory(volumeName, bucketName, keyName);
+      OzoneKeyDetails keyInfoDetails = checksumClient.getKeyDetails(volumeName, bucketName, keyName);
+      Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicas =
+          checksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName);
+      Map<OmKeyLocationInfo, Map<DatanodeDetails, OzoneInputStream>> replicasWithoutChecksum =
+          noChecksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName);
+
+      ObjectNode result = JsonUtils.createObjectNode(null);
+      result.put(JSON_PROPERTY_FILE_NAME, volumeName + "/" + bucketName + "/" + keyName);
+      result.put(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize());
+
+      ArrayNode blocks = JsonUtils.createArrayNode();
+      downloadReplicasAndCreateManifest(keyName, replicas, replicasWithoutChecksum, dir, blocks);
+      result.set(JSON_PROPERTY_FILE_BLOCKS, blocks);
+
+      String prettyJson = JsonUtils.toJsonStringWithDefaultPrettyPrinter(result);
+
+      String manifestFileName = keyName + "_manifest";
+      System.out.println("Writing manifest file : " + manifestFileName);
+      File manifestFile = new File(dir, manifestFileName);
+      Files.write(manifestFile.toPath(), prettyJson.getBytes(StandardCharsets.UTF_8));
+    } finally {
+      newClient.close();
+    }
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/Replicas.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import org.apache.hadoop.hdds.cli.DebugSubcommand;
+import org.kohsuke.MetaInfServices;
+import picocli.CommandLine;
+
+/**
+ * Handler to print information about replicas.
+ */
+@CommandLine.Command(
+    name = "replicas",
+    description = "Run various debug tools across all replicas",
+    subcommands = {
+        ReplicasVerify.class
+    })
+@MetaInfServices(DebugSubcommand.class)
+public class Replicas implements DebugSubcommand {
+
+}

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ReplicasVerify.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import picocli.CommandLine;
+
+/**
+ * Verify replicas command.
+ */
+
+@CommandLine.Command(
+    name = "verify",
+    description = "Run various debug tools to verify data across replicas",
+    subcommands = {
+        ReadReplicas.class,
+        FindMissingPadding.class
+    })
+public class ReplicasVerify {
+
+}