KMS-700: End to End testing.

README.md

@@ -480,8 +480,11 @@ export bamboo_EDL_UID=[edl user id]
 export bamboo_EDL_PASSWORD=[edl password]
 export bamboo_EDL_CLIENT_ID=[edl oauth client id]
 export bamboo_CMR_BASE_URL=[cmr base url]
-export bamboo_CMR_WRITER_TOKEN=[optional bearer token for CMR metadata writeback]
+export bamboo_CMR_SYSTEM_TOKEN_PARAMETER_NAME=[optional SSM parameter containing the CMR system token]
+export bamboo_CMR_WRITER_TOKEN=[optional complete bearer authorization value used as fallback]
 export bamboo_CMR_WRITEBACK_PROVIDERS=[optional provider id, comma-separated list, or ALL]
+export bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS=[true|false; defaults to false]
+export bamboo_CMR_WRITEBACK_VALIDATE_UMM_C=[true|false; defaults to false]
 export bamboo_CORS_ORIGIN=[comma separated list of cors origins]
 export bamboo_RDF4J_CONTAINER_MEMORY_LIMIT=[7168 for sit|uat, 14336 for prod]
 export bamboo_RDF4J_INSTANCE_TYPE=["M5.LARGE" for sit|uat, "R5.LARGE" for prod]
@@ -495,8 +498,12 @@ export bamboo_KMS_REDIS_NODE_TYPE=[for example cache.t3.micro]
 ```
 Notes:
 - `bamboo_CMR_BASE_URL` is required. KMS no longer falls back to `https://cmr.earthdata.nasa.gov` during deploy or synth.
-- Leave `bamboo_CMR_WRITER_TOKEN` empty to disable live CMR writeback.
+- When configured, `bamboo_CMR_SYSTEM_TOKEN_PARAMETER_NAME` is the primary source for the CMR
+  authorization value. `bamboo_CMR_WRITER_TOKEN` is used only as a fallback and must include the
+  `Bearer` prefix.
 - Leave `bamboo_CMR_WRITEBACK_PROVIDERS` empty to disable provider rollout for CMR writeback.
+- Set `bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS` and `bamboo_CMR_WRITEBACK_VALIDATE_UMM_C`
+  to `true` to reject writebacks that still fail CMR keyword or UMM-C validation.
 - If you are not deploying into an existing API Gateway, set `bamboo_EXISTING_API_ID` and `bamboo_ROOT_RESOURCE_ID` to empty strings.
 - If `bamboo_RDF4J_BACKUP_VAULT_NAME` is set, `SnapshotStack` imports that existing backup vault. This is useful when `rdf4jSnapshotStack` is being recreated after an RDF4J recovery event and you need the new stack to reuse an existing vault instead of trying to create the same vault name again.
 - If `bamboo_RDF4J_BACKUP_VAULT_NAME` is not set, `SnapshotStack` creates the default `rdf4j-backup-vault`.

bin/deploy-bamboo.sh

@@ -67,6 +67,8 @@ dockerRun() {
         --env "CMR_SYSTEM_TOKEN_PARAMETER_NAME=${bamboo_CMR_SYSTEM_TOKEN_PARAMETER_NAME}" \
         --env "CMR_WRITER_TOKEN=${bamboo_CMR_WRITER_TOKEN:-}" \
         --env "CMR_WRITEBACK_PROVIDERS=${bamboo_CMR_WRITEBACK_PROVIDERS:-}" \
+        --env "CMR_WRITEBACK_VALIDATE_KEYWORDS=${bamboo_CMR_WRITEBACK_VALIDATE_KEYWORDS:-false}" \
+        --env "CMR_WRITEBACK_VALIDATE_UMM_C=${bamboo_CMR_WRITEBACK_VALIDATE_UMM_C:-false}" \
         --env "METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY=${bamboo_METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY:-5}" \
         --env "BLOCK_PUBLISH_ON_KEYWORD_DIFF_FAILURE=${bamboo_BLOCK_PUBLISH_ON_KEYWORD_DIFF_FAILURE:-false}" \
         --env "KEYWORD_SYNC_ALARM_EMAILS=${bamboo_KEYWORD_SYNC_ALARM_EMAILS:-}" \

cdk/app/lib/CmrEventProcessingStack.ts

@@ -18,6 +18,8 @@ export interface CmrEventProcessingStackProps extends cdk.StackProps {
   metadataCorrectionServiceReservedConcurrency?: string
   cmrWriterToken?: string
   cmrWritebackProviders?: string
+  cmrWritebackValidateKeywords?: string
+  cmrWritebackValidateUmmC?: string
   redisEnabled?: string
   redisHost?: string
   redisPort?: string
@@ -69,6 +71,8 @@ export class CmrEventProcessingStack extends cdk.Stack {
         props.metadataCorrectionServiceReservedConcurrency,
       cmrWriterToken: props.cmrWriterToken,
       cmrWritebackProviders: props.cmrWritebackProviders,
+      cmrWritebackValidateKeywords: props.cmrWritebackValidateKeywords,
+      cmrWritebackValidateUmmC: props.cmrWritebackValidateUmmC,
       prefix: props.prefix,
       redisEnabled: props.redisEnabled,
       redisHost: props.redisHost,

cdk/app/lib/KmsStack.ts

@@ -20,6 +20,8 @@ import { VpcSetup } from './helper/VpcSetup'
 export interface KmsStackProps extends cdk.StackProps {
   cmrSystemTokenParameterName?: string
   cmrWritebackProviders?: string
+  cmrWritebackValidateKeywords?: string
+  cmrWritebackValidateUmmC?: string
   cmrWriterToken?: string
   existingApiId: string | undefined
   keywordSyncAlarmEmails?: string[]
@@ -173,6 +175,8 @@ export class KmsStack extends cdk.Stack {
         CMR_SYSTEM_TOKEN_PARAMETER_NAME: props.cmrSystemTokenParameterName || '',
         CMR_WRITER_TOKEN: props.cmrWriterToken || '',
         CMR_WRITEBACK_PROVIDERS: props.cmrWritebackProviders || '',
+        CMR_WRITEBACK_VALIDATE_KEYWORDS: props.cmrWritebackValidateKeywords || '',
+        CMR_WRITEBACK_VALIDATE_UMM_C: props.cmrWritebackValidateUmmC || '',
         METADATA_CORRECTION_REQUESTS_TOPIC_ARN: metadataCorrectionRequestsTopicArn
       },
       prefix,

cdk/app/lib/helper/KmsLambdaFunctions.ts

@@ -24,6 +24,8 @@ interface LambdaFunctionsProps {
     CMR_SYSTEM_TOKEN_PARAMETER_NAME?: string;
     CMR_WRITER_TOKEN: string;
     CMR_WRITEBACK_PROVIDERS: string;
+    CMR_WRITEBACK_VALIDATE_KEYWORDS: string;
+    CMR_WRITEBACK_VALIDATE_UMM_C: string;
     METADATA_CORRECTION_REQUESTS_TOPIC_ARN?: string;
   };
   prefix: string;

cdk/app/lib/helper/MetadataCorrectionSetup.ts

@@ -22,6 +22,8 @@ interface MetadataCorrectionSetupProps {
   metadataCorrectionServiceReservedConcurrency?: string
   cmrWriterToken?: string
   cmrWritebackProviders?: string
+  cmrWritebackValidateKeywords?: string
+  cmrWritebackValidateUmmC?: string
   prefix: string
   redisEnabled?: string
   redisHost?: string
@@ -74,6 +76,8 @@ export class MetadataCorrectionSetup extends Construct {
       metadataCorrectionServiceReservedConcurrency,
       cmrWriterToken,
       cmrWritebackProviders,
+      cmrWritebackValidateKeywords,
+      cmrWritebackValidateUmmC,
       prefix,
       redisEnabled,
       redisHost,
@@ -152,6 +156,12 @@ export class MetadataCorrectionSetup extends Construct {
             : {}),
           CMR_WRITER_TOKEN: cmrWriterToken || '',
           ...(cmrWritebackProviders ? { CMR_WRITEBACK_PROVIDERS: cmrWritebackProviders } : {}),
+          ...(cmrWritebackValidateKeywords
+            ? { CMR_WRITEBACK_VALIDATE_KEYWORDS: cmrWritebackValidateKeywords }
+            : {}),
+          ...(cmrWritebackValidateUmmC
+            ? { CMR_WRITEBACK_VALIDATE_UMM_C: cmrWritebackValidateUmmC }
+            : {}),
           ...(redisEnabled ? { REDIS_ENABLED: redisEnabled } : {}),
           ...(redisHost ? { REDIS_HOST: redisHost } : {}),
           ...(redisPort ? { REDIS_PORT: redisPort } : {}),

cdk/bin/main.ts

@@ -183,6 +183,8 @@ async function main() {
     cmrSystemTokenParameterName: process.env.CMR_SYSTEM_TOKEN_PARAMETER_NAME || '',
     cmrWriterToken: process.env.CMR_WRITER_TOKEN || '',
     cmrWritebackProviders: process.env.CMR_WRITEBACK_PROVIDERS || '',
+    cmrWritebackValidateKeywords: process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS || '',
+    cmrWritebackValidateUmmC: process.env.CMR_WRITEBACK_VALIDATE_UMM_C || '',
     prefix,
     env,
     vpcId,
@@ -232,6 +234,8 @@ async function main() {
       process.env.METADATA_CORRECTION_SERVICE_RESERVED_CONCURRENCY || '',
     cmrWriterToken: process.env.CMR_WRITER_TOKEN || '',
     cmrWritebackProviders: process.env.CMR_WRITEBACK_PROVIDERS || '',
+    cmrWritebackValidateKeywords: process.env.CMR_WRITEBACK_VALIDATE_KEYWORDS || '',
+    cmrWritebackValidateUmmC: process.env.CMR_WRITEBACK_VALIDATE_UMM_C || '',
     env,
     prefix,
     redisEnabled: redisEnabledValue,

serverless/src/shared/Iso19115DomEditor.js

@@ -17,14 +17,34 @@ const blockScheme = (config) => (editor, correction) => editor.updateBlockNode(c
  */
 const leafScheme = (config) => (editor, correction) => editor.updateLeafNode(correction, config)
 
+/**
+ * Builds an ISO keyword path, using NONE only for gaps before a later populated level.
+ *
+ * @param {string[]} fieldKeys Ordered keyword hierarchy fields.
+ * @param {Object} keywordObject Keyword values keyed by hierarchy field.
+ * @returns {string} Serialized ISO keyword path.
+ */
+const buildKeywordPath = (fieldKeys, keywordObject = {}) => {
+  const values = fieldKeys.map((key) => String(keywordObject[key] || '').trim())
+  const lastValueIndex = values.reduce(
+    (lastIndex, value, index) => (value ? index : lastIndex),
+    -1
+  )
+
+  return values
+    .slice(0, lastValueIndex + 1)
+    .map((value) => value || 'NONE')
+    .join(' > ')
+}
+
 /**
  * Factory to generate standardized keyword block editors.
  * @param {string} type - The 'codeListValue' for the MD_KeywordTypeCode.
  * @param {Object} options - Configuration options.
  * @param {Array} [options.fieldKeys] - Array of keys representing the structure of the keyword object.
  * @param {Array} [options.matchKeys] - Array of keys used to match existing keywords.
  * @param {Function} [options.getValue] - Optional custom function to format the string value before saving.
- * @param {Array} [options.additionalPaths] - Optional array of XPath strings or objects for secondary sync.
+ * @param {Array<{path: string, getValue: Function}>} [options.additionalPaths] Secondary sync paths.
  * @param {string} [options.nodeXPath] - Optional custom XPath string to identify the keyword node, overrides default.
  */
 const createKeywordBlock = (type, {
@@ -56,29 +76,19 @@ const createKeywordBlock = (type, {
       fieldPath: ({ node, editor }) => (editor.selectNodes('./gmx:Anchor', node).length > 0 ? 'gmx:Anchor' : 'gco:CharacterString'),
       source: {
         type: 'computed',
-        getValue: getValue || (({ correction }) => fieldKeys
-          .map((k) => correction.newKeywordObject[k] || 'NONE')
-          .join(' > ')
-        )
+        getValue: getValue || (({ correction }) => buildKeywordPath(
+          fieldKeys,
+          correction.newKeywordObject
+        ))
       }
     },
-    // Dynamically add secondary paths for synchronization
-    // Each path can be a string or an object with { path, getValue }
-    ...additionalPaths.map((pathConfig) => {
-      const isObject = typeof pathConfig === 'object' && pathConfig.path
-      const path = isObject ? pathConfig.path : pathConfig
-      const pathGetValue = isObject ? pathConfig.getValue : null
-
-      return {
-        fieldPath: path,
-        source: {
-          type: 'computed',
-          getValue: pathGetValue || getValue || (({ correction }) => fieldKeys
-            .map((k) => correction.newKeywordObject[k] || 'NONE')
-            .join(' > '))
-        }
+    ...additionalPaths.map(({ path, getValue: pathGetValue }) => ({
+      fieldPath: path,
+      source: {
+        type: 'computed',
+        getValue: pathGetValue
       }
-    })
+    }))
   ]
 })
 
@@ -123,9 +133,15 @@ const createProductLevelIdEditor = () => leafScheme({
   },
   delete: [
     // Remove the entire processingLevel parent wrapper (in identificationInfo)
-    { path: '//gmd:identificationInfo//gmd:processingLevel[gmd:MD_Identifier/gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]' },
+    {
+      path: '//gmd:identificationInfo//gmd:processingLevel[gmd:MD_Identifier/gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]',
+      matchValuePath: 'gmd:MD_Identifier/gmd:code/gco:CharacterString'
+    },
     // Remove the entire processingLevelCode parent wrapper (in contentInfo)
-    { path: '//gmd:contentInfo//gmd:processingLevelCode[gmd:MD_Identifier/gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]' }
+    {
+      path: '//gmd:contentInfo//gmd:processingLevelCode[gmd:MD_Identifier/gmd:codeSpace/gco:CharacterString="gov.nasa.esdis.umm.processinglevelid"]',
+      matchValuePath: 'gmd:MD_Identifier/gmd:code/gco:CharacterString'
+    }
   ],
   replace: [
     {
@@ -254,7 +270,7 @@ export const ISO_19115_SCHEME_EDITORS = {
     // Otherwise use split format (NSIDC/NOAA style)
     additionalPaths: [
       {
-        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:instrument/eos:EOS_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:code/gco:CharacterString',
+        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation//gmi:instrument/eos:EOS_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:code/gco:CharacterString',
         getValue: ({ correction, node }) => {
           const existingValue = node?.textContent || ''
           if (existingValue.includes(' > ')) {
@@ -268,7 +284,7 @@ export const ISO_19115_SCHEME_EDITORS = {
         }
       },
       {
-        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:instrument/gmi:MI_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:code/gco:CharacterString',
+        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation//gmi:instrument/gmi:MI_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:code/gco:CharacterString',
         getValue: ({ correction, node }) => {
           const existingValue = node?.textContent || ''
           if (existingValue.includes(' > ')) {
@@ -282,7 +298,7 @@ export const ISO_19115_SCHEME_EDITORS = {
         }
       },
       {
-        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:instrument/eos:EOS_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:description/gco:CharacterString',
+        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation//gmi:instrument/eos:EOS_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:description/gco:CharacterString',
         getValue: ({ correction, node, editor }) => {
           const codeNode = editor.selectNodes('../gmd:code/gco:CharacterString', node.parentNode)[0]
           const codeValue = codeNode?.textContent || ''
@@ -294,7 +310,7 @@ export const ISO_19115_SCHEME_EDITORS = {
         }
       },
       {
-        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:instrument/gmi:MI_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:description/gco:CharacterString',
+        path: '//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation//gmi:instrument/gmi:MI_Instrument/gmi:identifier/gmd:MD_Identifier/gmd:description/gco:CharacterString',
         getValue: ({ correction, node, editor }) => {
           const codeNode = editor.selectNodes('../gmd:code/gco:CharacterString', node.parentNode)[0]
           const codeValue = codeNode?.textContent || ''

serverless/src/shared/Iso19115MetadataPathEditor.js

@@ -171,7 +171,15 @@ export class Iso19115MetadataPathEditor extends XmlMetadataPathEditor {
         if (config.delete && config.delete.length > 0) {
           config.delete.forEach((delConfig) => {
             const nodesToDelete = this.selectNodes(delConfig.path, this.document)
-            nodesToDelete.forEach((node) => {
+            const matchingNodes = delConfig.matchValuePath
+              ? nodesToDelete.filter((node) => {
+                const valueNode = this.selectNodes(`./${delConfig.matchValuePath}`, node)[0]
+
+                return (valueNode?.textContent || '').toLowerCase().trim() === oldVal
+              })
+              : nodesToDelete
+
+            matchingNodes.forEach((node) => {
               if (node && node.parentNode) {
                 node.parentNode.removeChild(node)
               }
@@ -329,9 +337,16 @@ export class Iso19115MetadataPathEditor extends XmlMetadataPathEditor {
             // Check if we found a platform or instrument node
             if (localName === 'EOS_Platform' || localName === 'MI_Platform'
           || localName === 'EOS_Instrument' || localName === 'MI_Instrument') {
-              // Found the acquisition wrapper node, remove it entirely
-              if (currentNode.parentNode) {
-                currentNode.parentNode.removeChild(currentNode)
+              // Remove the property wrapper with its value to avoid leaving
+              // empty gmi:platform or gmi:instrument elements.
+              const propertyNode = currentNode.parentNode
+              const nodeToDelete = propertyNode
+                && (propertyNode.localName === 'platform' || propertyNode.localName === 'instrument')
+                ? propertyNode
+                : currentNode
+
+              if (nodeToDelete.parentNode) {
+                nodeToDelete.parentNode.removeChild(nodeToDelete)
               }
 
               break
@@ -371,8 +386,15 @@ export class Iso19115MetadataPathEditor extends XmlMetadataPathEditor {
           let currentNode = identifier.parentNode
           while (currentNode) {
             if (currentNode.localName === 'MI_Operation') {
-              if (currentNode.parentNode) {
-                currentNode.parentNode.removeChild(currentNode)
+              // Remove the operation property with its value so the
+              // acquisition block does not retain an empty gmi:operation.
+              const propertyNode = currentNode.parentNode
+              const nodeToDelete = propertyNode?.localName === 'operation'
+                ? propertyNode
+                : currentNode
+
+              if (nodeToDelete.parentNode) {
+                nodeToDelete.parentNode.removeChild(nodeToDelete)
               }
 
               break