HDDS-15239. Support persist StorageType on Datanode Container YAML

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -243,6 +243,7 @@ public final class OzoneConsts {
   public static final String ORIGIN_NODE_ID = "originNodeId";
   public static final String SCHEMA_VERSION = "schemaVersion";
   public static final String REPLICA_INDEX = "replicaIndex";
+  public static final String CONTAINER_STORAGE_TYPE = "storageType";
 
   // Supported .container datanode schema versions.
   // Since containers in older schema versions are currently not reformatted to

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java

@@ -42,6 +42,7 @@
 import java.util.regex.Pattern;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.fs.SpaceUsageSource;
@@ -247,11 +248,8 @@ public static void verifyContainerFileChecksum(ContainerData containerData,
     if (enabled) {
       String storedChecksum = containerData.getContainerFileChecksum();
 
-      Yaml yaml = ContainerDataYaml.getYamlForContainerType(
-          containerData.getContainerType(),
-          containerData instanceof KeyValueContainerData &&
-              ((KeyValueContainerData)containerData).getReplicaIndex() > 0);
-      containerData.computeAndSetContainerFileChecksum(yaml);
+      // Compute checksum (storageTypeis automatically excluded for rollback compatibility)
+      containerData.computeAndSetContainerFileChecksum();
       String computedChecksum = containerData.getContainerFileChecksum();
 
       if (storedChecksum == null || !storedChecksum.equals(computedChecksum)) {
@@ -375,7 +373,7 @@ public static void assertSpaceAvailability(long containerId, HddsVolume volume,
           + currentUsage + ", minimum free space spared="  + spared, DISK_OUT_OF_SPACE);
     }
   }
-  
+
   public static long getPendingDeletionBytes(ContainerData containerData) {
     if (containerData.getContainerType()
         .equals(ContainerProtos.ContainerType.KeyValueContainer)) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java

@@ -43,15 +43,18 @@
 import java.util.Optional;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.ratis.util.Preconditions;
 import org.yaml.snakeyaml.Yaml;
 
+
 /**
  * ContainerData is the in-memory representation of container metadata and is
  * represented on disk by the .container file.
@@ -114,6 +117,7 @@ public abstract class ContainerData {
   public static final Charset CHARSET_ENCODING = StandardCharsets.UTF_8;
   public static final String ZERO_CHECKSUM = new String(new byte[64],
       CHARSET_ENCODING);
+  private StorageType storageType;
 
   // Common Fields need to be stored in .container file.
   protected static final List<String> YAML_FIELDS =
@@ -162,6 +166,7 @@ protected ContainerData(ContainerData source) {
         source.getLayoutVersion(), source.getMaxSize(),
         source.getOriginPipelineId(), source.getOriginNodeId());
     replicaIndex = source.getReplicaIndex();
+    storageType = source.getStorageType();
   }
 
   /**
@@ -284,6 +289,14 @@ public Map<String, String> getMetadata() {
     return Collections.unmodifiableMap(this.metadata);
   }
 
+  public void setStorageType(StorageType type) {
+    storageType = type;
+  }
+
+  public StorageType getStorageType() {
+    return storageType;
+  }
+
   /**
    * Set metadata.
    * We should hold the container lock before updating the metadata as this
@@ -516,22 +529,32 @@ public String getOriginNodeId() {
   }
 
   /**
-   * Compute the checksum for ContainerData using the specified Yaml (based
-   * on ContainerType) and set the checksum.
+   * Compute the checksum for ContainerData and set the checksum.
    *
    * Checksum of ContainerData is calculated by setting the
    * {@link ContainerData#checksum} field to a 64-byte array with all 0's -
    * {@link ContainerData#ZERO_CHECKSUM}. After the checksum is calculated,
    * the checksum field is updated with this value.
    *
-   * @param yaml Yaml for ContainerType to get the ContainerData as Yaml String
    * @throws IOException
    */
-  public void computeAndSetContainerFileChecksum(Yaml yaml) throws IOException {
+  public void computeAndSetContainerFileChecksum() throws IOException {
     // Set checksum to dummy value - 0 byte array, to calculate the checksum
     // of rest of the data.
     this.checksum = ZERO_CHECKSUM;
 
+    // Create Yaml for checksum calculation that excludes storageType for backward compatibility
+    boolean withReplicaIndex = this instanceof KeyValueContainerData &&
+        ((KeyValueContainerData) this).getReplicaIndex() > 0;
+    // IMPORTANT: Pass null for storageType to ensure rollback compatibility.
+    // Ozone calculates checksum by reading the YAML file into a Java object, then converting
+    // it back to YAML string (not directly using the YAML file content).
+    // If storageType participates in checksum calculation, when rolling back to older versions that don't
+    // support storageType, they would recalculate checksum without storageType field
+    // (as it's unknown to them), causing checksum mismatch and validation failure.
+    Yaml yaml = ContainerDataYaml.getYamlForContainerType(
+        this.containerType, withReplicaIndex, null);
+
     // Dump yaml data into a string to compute its checksum
     String containerDataYamlStr = yaml.dump(this);
 
@@ -588,7 +611,7 @@ public boolean needsDataChecksum() {
    * - writeCount += 1 (one operation)
    * - usedSpace/committedBytes NOT updated here (delta handled separately)
    * - blockBytes NOT updated here (delta=4 handled by incrementBlockBytes)
-   * 
+   *
    * @param bytesWritten Number of bytes in the I/O operation
    * @param overwrite Whether this is an overwrite operation
    */

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.common.impl;
 
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_STORAGE_TYPE;
 import static org.apache.hadoop.ozone.OzoneConsts.REPLICA_INDEX;
 import static org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData.KEYVALUE_YAML_TAG;
 
@@ -31,6 +32,8 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -70,12 +73,18 @@ private ContainerDataYaml() {
   /**
    * Creates a .container file in yaml format.
    */
-  public static void createContainerFile(ContainerData containerData, File containerFile) throws IOException {
-    // Create Yaml for given container type
-    final Yaml yaml = getYamlForContainerType(containerData.getContainerType(), containerData.getReplicaIndex() > 0);
+  public static void createContainerFile(ContainerData containerData,
+      File containerFile) throws IOException {
+    boolean withReplicaIndex =
+        containerData instanceof KeyValueContainerData
+            && ((KeyValueContainerData) containerData).getReplicaIndex() > 0;
+    StorageType storageType =
+        containerData instanceof KeyValueContainerData
+            ? containerData.getStorageType() : null;
+    Yaml yaml = getYamlForContainerType(containerData.getContainerType(),
+        withReplicaIndex, storageType);
     // Compute Checksum and update ContainerData
-    containerData.computeAndSetContainerFileChecksum(yaml);
-
+    containerData.computeAndSetContainerFileChecksum();
     // Write the ContainerData with checksum to Yaml file.
     YamlUtils.dump(yaml, containerData, containerFile, LOG);
   }
@@ -157,6 +166,12 @@ public static ContainerData readContainer(InputStream input)
   public static Yaml getYamlForContainerType(ContainerType containerType,
       boolean withReplicaIndex)
       throws StorageContainerException {
+    return getYamlForContainerType(containerType, withReplicaIndex, null);
+  }
+
+  public static Yaml getYamlForContainerType(ContainerType containerType,
+      boolean withReplicaIndex, StorageType storageType)
+      throws StorageContainerException {
     PropertyUtils propertyUtils = new PropertyUtils();
     propertyUtils.setBeanAccess(BeanAccess.FIELD);
     propertyUtils.setAllowReadOnlyProperties(true);
@@ -168,6 +183,10 @@ public static Yaml getYamlForContainerType(ContainerType containerType,
         yamlFields = new ArrayList<>(yamlFields);
         yamlFields.add(REPLICA_INDEX);
       }
+      if (storageType != null) {
+        yamlFields = new ArrayList<>(yamlFields);
+        yamlFields.add(CONTAINER_STORAGE_TYPE);
+      }
       Representer representer = new ContainerDataRepresenter(yamlFields);
       representer.setPropertyUtils(propertyUtils);
       representer.addClassTag(
@@ -284,6 +303,12 @@ public Object construct(Node node) {
           kvData.setReplicaIndex(
               ((Long) replicaIndex).intValue());
         }
+        String storageTypeString = (String) nodes.get(CONTAINER_STORAGE_TYPE);
+        StorageType storageType = StringUtils.isEmpty(storageTypeString) ?
+            null : StorageType.valueOf(storageTypeString);
+        if (storageType != null) {
+          kvData.setStorageType(storageType);
+        }
         return kvData;
       }
     }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import net.jcip.annotations.Immutable;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
@@ -114,12 +115,12 @@ public StorageType getStorageType() {
     return storageType;
   }
 
-  private StorageTypeProto getStorageTypeProto() throws IOException {
+  private StorageTypeProto getStorageTypeProto() throws IllegalArgumentException {
     return getStorageTypeProto(getStorageType());
   }
 
   public static StorageTypeProto getStorageTypeProto(StorageType type)
-      throws IOException {
+      throws IllegalArgumentException {
     StorageTypeProto storageTypeProto;
     switch (type) {
     case SSD:
@@ -138,12 +139,12 @@ public static StorageTypeProto getStorageTypeProto(StorageType type)
       storageTypeProto = StorageTypeProto.RAM_DISK;
       break;
     default:
-      throw new IOException("Illegal Storage Type specified");
+      throw new IllegalArgumentException("Illegal Storage Type specified");
     }
     return storageTypeProto;
   }
 
-  public long getReserved() { 
+  public long getReserved() {
     return reserved;
   }
 
@@ -155,8 +156,8 @@ public long getFsAvailable() {
     return fsAvailable;
   }
 
-  private static StorageType getStorageType(StorageTypeProto proto) throws
-      IOException {
+  public static StorageType getStorageType(StorageTypeProto proto) throws
+      IllegalArgumentException {
     StorageType storageType;
     switch (proto) {
     case SSD:
@@ -175,7 +176,7 @@ private static StorageType getStorageType(StorageTypeProto proto) throws
       storageType = StorageType.RAM_DISK;
       break;
     default:
-      throw new IOException("Illegal Storage Type specified");
+      throw new IllegalArgumentException("Illegal Storage Type specified");
     }
     return storageType;
   }
@@ -184,9 +185,14 @@ private static StorageType getStorageType(StorageTypeProto proto) throws
    * Returns the StorageReportProto protoBuf message for the Storage Location
    * report.
    * @return StorageReportProto
-   * @throws IOException In case, the storage type specified is invalid.
+   * @throws IllegalArgumentException In case, the storage type specified is invalid.
    */
-  public StorageReportProto getProtoBufMessage() throws IOException {
+  public StorageReportProto getProtoBufMessage() throws IllegalArgumentException, IOException {
+    return getProtoBufMessage(null);
+  }
+
+  public StorageReportProto getProtoBufMessage(ConfigurationSource conf)
+      throws IOException {
     StorageReportProto.Builder srb = StorageReportProto.newBuilder();
     return srb.setStorageUuid(getId())
         .setCapacity(getCapacity())
@@ -207,10 +213,10 @@ public StorageReportProto getProtoBufMessage() throws IOException {
    * Returns the MetadataStorageReportProto protoBuf message for the
    * Storage Location report.
    * @return MetadataStorageReportProto
-   * @throws IOException In case, the storage type specified is invalid.
+   * @throws IllegalArgumentException In case, the storage type specified is invalid.
    */
   public MetadataStorageReportProto getMetadataProtoBufMessage()
-      throws IOException {
+      throws IllegalArgumentException {
     MetadataStorageReportProto.Builder srb =
         MetadataStorageReportProto.newBuilder();
     return srb.setCapacity(getCapacity())
@@ -226,11 +232,11 @@ public MetadataStorageReportProto getMetadataProtoBufMessage()
    * Returns the StorageLocationReport from the protoBuf message.
    * @param report SCMStorageReport
    * @return StorageLocationReport
-   * @throws IOException in case of invalid storage type
+   * @throws IllegalArgumentException in case of invalid storage type
    */
 
   public static StorageLocationReport getFromProtobuf(StorageReportProto report)
-      throws IOException {
+      throws IllegalArgumentException {
     StorageLocationReport.Builder builder = StorageLocationReport.newBuilder();
     builder.setId(report.getStorageUuid())
         .setStorageLocation(report.getStorageLocation());

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java

@@ -56,7 +56,7 @@ protected long getReportFrequency() {
   }
 
   @Override
-  protected NodeReportProto getReport() throws IOException {
+  protected NodeReportProto getReport() throws IOException, IllegalArgumentException {
     return getContext().getParent().getContainer().getNodeReport();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java

@@ -83,7 +83,7 @@ private void publishReport() {
       } else {
         context.refreshFullReport(report);
       }
-    } catch (IOException e) {
+    } catch (IOException | IllegalArgumentException e) {
       LOG.error("Exception while publishing report.", e);
     }
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -677,7 +677,7 @@ public ContainerController getController() {
    * Returns node report of container storage usage.
    */
   public StorageContainerDatanodeProtocolProtos.NodeReportProto getNodeReport()
-          throws IOException {
+          throws IOException, IllegalArgumentException {
     StorageLocationReport[] reports = volumeSet.getStorageReport();
     StorageContainerDatanodeProtocolProtos.NodeReportProto.Builder nrb
             = StorageContainerDatanodeProtocolProtos.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestKeyValueContainerData.java

@@ -25,6 +25,7 @@
 
 import java.util.UUID;
 import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -94,8 +95,9 @@ public void testKeyValueData(ContainerTestVersionInfo versionInfo) {
     kvData.incrPendingDeletionBlocks(1, 256);
     kvData.setSchemaVersion(
         VersionedDatanodeFeatures.SchemaV3.chooseSchemaVersion(conf));
-    long expectedDataHash =  1234L;
+    long expectedDataHash = 1234L;
     kvData.setDataChecksum(expectedDataHash);
+    kvData.setStorageType(StorageType.SSD);
 
     assertEquals(state, kvData.getState());
     assertEquals(containerDBType, kvData.getContainerDBType());
@@ -110,12 +112,14 @@ public void testKeyValueData(ContainerTestVersionInfo versionInfo) {
     assertEquals(VersionedDatanodeFeatures.SchemaV3.chooseSchemaVersion(conf),
         kvData.getSchemaVersion());
     assertEquals(expectedDataHash, kvData.getDataChecksum());
+    assertEquals(StorageType.SSD, kvData.getStorageType());
 
     KeyValueContainerData newKvData = new KeyValueContainerData(kvData);
     assertEquals(kvData.getReplicaIndex(), newKvData.getReplicaIndex());
     assertEquals(0, newKvData.getNumPendingDeletionBlocks());
     assertEquals(0, newKvData.getDeleteTransactionId());
     assertEquals(kvData.getSchemaVersion(), newKvData.getSchemaVersion());
+    assertEquals(kvData.getStorageType(), newKvData.getStorageType());
   }
 
   @ContainerTestVersionInfo.ContainerTest

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java

@@ -599,8 +599,8 @@ private KeyValueContainerData newKvData() throws IOException {
     // from the container file and calculated at run time.
     Yaml yaml = ContainerDataYaml.getYamlForContainerType(
             kvData.getContainerType(),
-        kvData.getReplicaIndex() > 0);
-    kvData.computeAndSetContainerFileChecksum(yaml);
+        kvData.getReplicaIndex() > 0, kvData.getStorageType());
+    kvData.computeAndSetContainerFileChecksum();
 
     KeyValueContainerUtil.parseKVContainerData(kvData, conf);