From 99002020422302180b37dde6fc61d1e07e5b59a3 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 25 Feb 2026 09:25:36 -0500 Subject: [PATCH 01/30] Integrate BULKI serializer to save and restore server checkpoints --- src/server/pdc_server.c | 846 +++++++++++++++++++++++++++------------- 1 file changed, 567 insertions(+), 279 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index abe35434d..9b0ae94c1 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -74,6 +74,9 @@ rocksdb_t *rocksdb_g; sqlite3 *sqlite3_db_g; #endif +#include "bulki.h" +#include "bulki_serde.h" + // Check how long PDC has run every OP_INTERVAL operations #define PDC_CHECKPOINT_CHK_OP_INTERVAL 2000 // Checkpoint every INTERVAL_SEC second and at least OP_INTERVAL operations @@ -1191,12 +1194,17 @@ PDC_Server_checkpoint() HashTablePair pair; char checkpoint_file[ADDR_MAX], checkpoint_file_local[ADDR_MAX], cmd[4096]; HashTableIterator hash_table_iter; - char * checkpoint; +// char * checkpoint; char * env_char; - uint64_t checkpoint_size; +// uint64_t checkpoint_size; bool use_tmpfs = false; FILE * file; + // BULKI-specific variables + BULKI *checkpoint_bulki = NULL; + size_t serialized_size = 0; + void * serialized_buffer = NULL; + #ifdef PDC_TIMING // Timing struct timeval pdc_timer_start; @@ -1223,31 +1231,42 @@ PDC_Server_checkpoint() if (pdc_server_rank_g == 0) LOG_INFO("Checkpoint file [%s]\n", checkpoint_file); - if (use_tmpfs) - file = fopen(checkpoint_file_local, "w+"); - else - file = fopen(checkpoint_file, "w+"); - - if (file == NULL) - PGOTO_ERROR(FAIL, "Checkpoint file open error"); + // Initialize BULKI structure - estimate initial capacity + checkpoint_bulki = BULKI_init(10); // todo: this should be 4 - // Checkpoint containers + // ========== Section 1: Checkpoint Containers ========== n_entry = hash_table_num_entries(container_hash_table_g); - fwrite(&n_entry, sizeof(int), 1, file); + + BULKI_Entity *containers_array = empty_BULKI_Array_Entity(); hash_table_iterate(container_hash_table_g, &hash_table_iter); while (n_entry != 0 && hash_table_iter_has_more(&hash_table_iter)) { pair = hash_table_iter_next(&hash_table_iter); cont_head = pair.value; + BULKI *container_entry = BULKI_init(2); + hash_key = PDC_get_hash_by_name(cont_head->cont_name); - fwrite(&hash_key, sizeof(uint32_t), 1, file); - fwrite(cont_head, sizeof(pdc_cont_hash_table_entry_t), 1, file); + BULKI_put(container_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + + // Store the container structure as binary blob + BULKI_put(container_entry, + BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(containers_array, container_entry); } - // DHT + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("containers", PDC_STRING), + containers_array); + + // ========== Section 2: Checkpoint Metadata Hash Table ========== n_entry = hash_table_num_entries(metadata_hash_table_g); - fwrite(&n_entry, sizeof(int), 1, file); + + BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity(); hash_table_iterate(metadata_hash_table_g, &hash_table_iter); @@ -1255,83 +1274,218 @@ PDC_Server_checkpoint() pair = hash_table_iter_next(&hash_table_iter); head = pair.value; - fwrite(&head->n_obj, sizeof(int), 1, file); + BULKI *hash_entry = BULKI_init(3); + + // Store number of objects + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("n_obj", PDC_STRING), + BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); + hash_key = PDC_get_hash_by_name(head->metadata->obj_name); - fwrite(&hash_key, sizeof(uint32_t), 1, file); + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + + // Array of metadata objects + BULKI_Entity *metadata_objs_array = empty_BULKI_Array_Entity(); // Iterate every metadata structure in current entry DL_FOREACH(head->metadata, elt) { - // Write entire metadata structure - fwrite(elt, sizeof(pdc_metadata_t), 1, file); + BULKI *metadata_obj = BULKI_init(4); // todo: maybe this should be 3 - // Write kv tags + // Store metadata structure + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("metadata", PDC_STRING), + BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); + + // ========== KV Tags ========== DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); - fwrite(&n_kvtag, sizeof(int), 1, file); + + BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity(); DL_FOREACH(elt->kvtag_list_head, kvlist_elt) { + BULKI *kvtag_entry = BULKI_init(4); + key_len = strlen(kvlist_elt->kvtag->name) + 1; - fwrite(&key_len, sizeof(int), 1, file); - fwrite(kvlist_elt->kvtag->name, key_len, 1, file); - fwrite(&kvlist_elt->kvtag->size, sizeof(uint32_t), 1, file); - fwrite(&kvlist_elt->kvtag->type, sizeof(int8_t), 1, file); - fwrite(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, 1, file); + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("key", PDC_STRING), + BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("size", PDC_STRING), + BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("type", PDC_STRING), + BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); + + BULKI_put(kvtag_entry, + BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, + PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(kvtags_array, kvtag_entry); } - // Write region info + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("kvtags", PDC_STRING), + kvtags_array); + + // ========== Storage Regions ========== n_region = 0; DL_COUNT(elt->storage_region_list_head, region_elt, n_region); - fwrite(&n_region, sizeof(int), 1, file); + + BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); if (n_region > 0) { n_write_region = 0; DL_FOREACH(elt->storage_region_list_head, region_elt) { - fwrite(region_elt, sizeof(region_list_t), 1, file); - n_write_region++; + BULKI *region_entry = BULKI_init(3); + + // Store region structure + BULKI_put(region_entry, + BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + + // Store histogram if present int has_hist = 0; if (region_elt->region_hist != NULL) has_hist = 1; - fwrite(&has_hist, sizeof(int), 1, file); + + BULKI_put(region_entry, + BULKI_singleton_ENTITY("has_hist", PDC_STRING), + BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); + if (has_hist == 1) { - fwrite(®ion_elt->region_hist->dtype, sizeof(int), 1, file); - fwrite(®ion_elt->region_hist->nbin, sizeof(int), 1, file); - fwrite(region_elt->region_hist->range, sizeof(double), - region_elt->region_hist->nbin * 2, file); - fwrite(region_elt->region_hist->bin, sizeof(uint64_t), region_elt->region_hist->nbin, - file); - fwrite(®ion_elt->region_hist->incr, sizeof(double), 1, file); + BULKI *histogram = BULKI_init(5); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("range", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->range, + region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("bin", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->bin, + region_elt->region_hist->nbin, PDC_UINT64, PDC_CLS_ARRAY)); + + BULKI_put(histogram, + BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); + + BULKI_put(region_entry, + BULKI_singleton_ENTITY("histogram", PDC_STRING), + BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } + + BULKI_ENTITY_append_BULKI(regions_array, region_entry); + n_write_region++; } if (n_write_region != n_region) LOG_ERROR("Error with number of regions\n"); } + + BULKI_put(metadata_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING), + regions_array); + + BULKI_ENTITY_append_BULKI(metadata_objs_array, metadata_obj); + metadata_size++; region_count += n_region; } // End for metadata entry linked list - } // End for hash table metadata entry - // Note data server region are managed by data server instead of metadata server + BULKI_put(hash_entry, + BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), + metadata_objs_array); + + BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); + } // End for hash table metadata entry + + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), + metadata_entries_array); + + // ========== Section 3: Data Server Regions ========== data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); - fwrite(&n_objs, sizeof(int), 1, file); + + BULKI_Entity *dataserver_regions_array = empty_BULKI_Array_Entity(); + DL_FOREACH(dataserver_region_g, region) { - fwrite(®ion->obj_id, sizeof(uint64_t), 1, file); + BULKI *dataserver_obj = BULKI_init(3); + + BULKI_put(dataserver_obj, + BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + DL_COUNT(region->region_storage_head, region_elt, n_region); - fwrite(&n_region, sizeof(int), 1, file); + + BULKI_Entity *ds_regions_array = empty_BULKI_Array_Entity(); DL_FOREACH(region->region_storage_head, region_elt) { - fwrite(region_elt, sizeof(region_list_t), 1, file); + BULKI *ds_region = BULKI_init(1); + BULKI_put(ds_region, + BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(ds_regions_array, ds_region); } + + BULKI_put(dataserver_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING), + ds_regions_array); + + BULKI_ENTITY_append_BULKI(dataserver_regions_array, dataserver_obj); } - transfer_request_metadata_query_checkpoint(&checkpoint, &checkpoint_size); - fwrite(&checkpoint_size, sizeof(uint64_t), 1, file); - fwrite(checkpoint, checkpoint_size, 1, file); + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), + dataserver_regions_array); + // ========== Section 4: Transfer Request Metadata Query ========== + char * query_checkpoint; + uint64_t query_checkpoint_size; + transfer_request_metadata_query_checkpoint(&query_checkpoint, &query_checkpoint_size); + + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("transfer_query", PDC_STRING), + BULKI_ENTITY(query_checkpoint, query_checkpoint_size, PDC_UINT8, PDC_CLS_ARRAY)); + + // ========== Serialize and Write ========== + serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); + + if (use_tmpfs) + file = fopen(checkpoint_file_local, "wb"); + else + file = fopen(checkpoint_file, "wb"); + + if (file == NULL) + PGOTO_ERROR(FAIL, "Checkpoint file open error"); + + // Write BULKI magic header for validation + uint32_t magic = 0x424C4B49; // "BLKI" in hex + fwrite(&magic, sizeof(uint32_t), 1, file); + + // Write serialized data + fwrite(serialized_buffer, serialized_size, 1, file); fclose(file); + // Clean up + BULKI_free(checkpoint_bulki, 1); + serialized_buffer = PDC_free(serialized_buffer); + if (use_tmpfs) { #ifdef PDC_TIMING gettimeofday(&pdc_timer_end_rank, 0); @@ -1358,8 +1512,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, - checkpoint_time_rank); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs (BULKI size: %zu bytes)\n", + metadata_size, region_count, checkpoint_time_rank, serialized_size); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1403,17 +1557,24 @@ PDC_Server_restart(char *filename) FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; - int n_entry, count, i, j, nobj = 0, all_nobj = 0, all_n_region, n_region, n_objs, total_region = 0, - n_kvtag, key_len; - int n_cont, all_cont; + int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; + int all_cont; pdc_metadata_t * metadata, *elt; region_list_t * region_list; pdc_hash_table_entry_head * entry; pdc_cont_hash_table_entry_t *cont_entry; uint32_t * hash_key; unsigned idx; - uint64_t checkpoint_size; - char * checkpoint_buf; + + // BULKI-specific variables + BULKI * checkpoint_bulki = NULL; + BULKI_Entity * containers_array = NULL; + BULKI_Entity * metadata_entries_array = NULL; + BULKI_Entity * dataserver_regions_array = NULL; + void * file_buffer = NULL; + size_t file_size; + uint32_t magic; + #ifdef PDC_TIMING double start = MPI_Wtime(); #endif @@ -1421,275 +1582,402 @@ PDC_Server_restart(char *filename) // init hash table ret_value = PDC_Server_init_hash_table(); if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "Error wtih PDC_Server_init_hash_table"); + PGOTO_ERROR(FAIL, "Error with PDC_Server_init_hash_table"); - FILE *file = fopen(filename, "r"); + // Read entire file into memory + FILE *file = fopen(filename, "rb"); if (file == NULL) PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); - if (fread(&n_cont, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_count\n"); - } - all_cont = n_cont; - while (n_cont > 0) { - hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); - if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for hash_key\n"); - } - total_mem_usage_g += sizeof(uint32_t); - - // Reconstruct hash table - cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); - total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); - if (fread(cont_entry, sizeof(pdc_cont_hash_table_entry_t), 1, file) != 1) { - LOG_ERROR("Read failed for cont_entry\n"); - } + fseek(file, 0, SEEK_END); + file_size = ftell(file); + fseek(file, 0, SEEK_SET); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&pdc_container_hash_table_mutex_g); -#endif - if (hash_table_insert(container_hash_table_g, hash_key, cont_entry) != 1) { - LOG_ERROR("Hash table insert failed\n"); - ret_value = FAIL; - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&pdc_container_hash_table_mutex_g); -#endif + file_buffer = PDC_malloc(file_size); + if (fread(file_buffer, file_size, 1, file) != 1) { + LOG_ERROR("Failed to read checkpoint file\n"); + fclose(file); + PGOTO_ERROR(FAIL, "Read failed for checkpoint file"); + } + fclose(file); - n_cont--; - } // End while + // Validate magic header + memcpy(&magic, file_buffer, sizeof(uint32_t)); + if (magic != 0x424C4B49) { + LOG_ERROR("Invalid checkpoint file magic header\n"); + PGOTO_ERROR(FAIL, "Invalid checkpoint format"); + } - if (fread(&n_entry, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_entry\n"); + // Deserialize BULKI + checkpoint_bulki = BULKI_deserialize((char *)file_buffer + sizeof(uint32_t)); + if (checkpoint_bulki == NULL) { + LOG_ERROR("Failed to deserialize checkpoint file\n"); + PGOTO_ERROR(FAIL, "Deserialization failed"); } - while (n_entry > 0) { - if (fread(&count, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for count\n"); - } - hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); - if (fread(hash_key, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for hash_key\n"); - } - total_mem_usage_g += sizeof(uint32_t); - - // Reconstruct hash table - entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); - entry->n_obj = 0; - entry->bloom = NULL; - entry->metadata = NULL; - // Init hash table metadata (w/ bloom) with first obj - PDC_Server_hash_table_list_init(entry, hash_key); - - metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); - for (i = 0; i < count; i++) { - if (fread(metadata + i, sizeof(pdc_metadata_t), 1, file) != 1) { - LOG_ERROR("Read failed for metadata\n"); - } + // ========== Section 1: Restore Containers ========== + containers_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("containers", PDC_STRING)); - (metadata + i)->storage_region_list_head = NULL; - (metadata + i)->region_lock_head = NULL; - (metadata + i)->region_map_head = NULL; - (metadata + i)->region_buf_map_head = NULL; - (metadata + i)->bloom = NULL; - (metadata + i)->prev = NULL; - (metadata + i)->next = NULL; - (metadata + i)->kvtag_list_head = NULL; - (metadata + i)->all_storage_region_distributed = 0; - - // Read kv tags - if (fread(&n_kvtag, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_kvtag\n"); - } - for (j = 0; j < n_kvtag; j++) { - pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); - kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); - if (fread(&key_len, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for key_len\n"); - } - kvtag_list->kvtag->name = PDC_malloc(key_len); - if (fread((void *)(kvtag_list->kvtag->name), key_len, 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->name\n"); - } - if (fread(&kvtag_list->kvtag->size, sizeof(uint32_t), 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->size\n"); - } - if (fread(&kvtag_list->kvtag->type, sizeof(int8_t), 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->type\n"); - } - kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); - if (fread(kvtag_list->kvtag->value, kvtag_list->kvtag->size, 1, file) != 1) { - LOG_ERROR("Read failed for kvtag_list->kvtag->value\n"); - } - DL_APPEND((metadata + i)->kvtag_list_head, kvtag_list); - } + if (containers_array != NULL && containers_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *cont_iter = Bent_iterator_init(containers_array, NULL, PDC_BULKI); + all_cont = containers_array->count; - if (fread(&n_region, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_region\n"); - } - if (n_region < 0) - PGOTO_ERROR(FAIL, "Checkpoint file region was less than 0"); + while (Bent_iterator_has_next_BULKI(cont_iter)) { + BULKI *container_entry = Bent_iterator_next_BULKI(cont_iter); - /* if (n_region == 0) */ - /* continue; */ + // Extract hash key + BULKI_Entity *hash_key_ent = BULKI_get(container_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); + memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); + total_mem_usage_g += sizeof(uint32_t); - total_region += n_region; + // Extract container data + BULKI_Entity *cont_data_ent = BULKI_get(container_entry, + BULKI_singleton_ENTITY("cont_data", PDC_STRING)); + cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); + memcpy(cont_entry, cont_data_ent->data, sizeof(pdc_cont_hash_table_entry_t)); + total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); - for (j = 0; j < n_region; j++) { - region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - if (fread(region_list, sizeof(region_list_t), 1, file) != 1) { - LOG_ERROR("Read failed for region_list\n"); - } +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_lock(&pdc_container_hash_table_mutex_g); +#endif + if (hash_table_insert(container_hash_table_g, hash_key, cont_entry) != 1) { + LOG_ERROR("Hash table insert failed\n"); + ret_value = FAIL; + } +#ifdef ENABLE_MULTITHREAD + hg_thread_mutex_unlock(&pdc_container_hash_table_mutex_g); +#endif + } + } - int has_hist = 0; - if (fread(&has_hist, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for has_list\n"); - } - if (has_hist == 1) { - region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); - if (fread(®ion_list->region_hist->dtype, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->dtype\n"); - } - if (fread(®ion_list->region_hist->nbin, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->nbin\n"); - } - if (region_list->region_hist->nbin == 0) { - LOG_ERROR("Checkpoint file histogram size is 0\n"); + // ========== Section 2: Restore Metadata ========== + metadata_entries_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); + + if (metadata_entries_array != NULL && metadata_entries_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *entry_iter = Bent_iterator_init(metadata_entries_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(entry_iter)) { + BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); + + // Extract n_obj + BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("n_obj", PDC_STRING)); + int count; + memcpy(&count, n_obj_ent->data, sizeof(int)); + + // Extract hash key + BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); + memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); + total_mem_usage_g += sizeof(uint32_t); + + // Reconstruct hash table entry + entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); + entry->n_obj = 0; + entry->bloom = NULL; + entry->metadata = NULL; + PDC_Server_hash_table_list_init(entry, hash_key); + + metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); + + // Extract metadata objects array + BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, + BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); + + if (metadata_objs_array != NULL && metadata_objs_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(metadata_objs_array, NULL, PDC_BULKI); + i = 0; + + while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { + BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); + + // Extract metadata structure + BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("metadata", PDC_STRING)); + memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); + + // Initialize pointers + (metadata + i)->storage_region_list_head = NULL; + (metadata + i)->region_lock_head = NULL; + (metadata + i)->region_map_head = NULL; + (metadata + i)->region_buf_map_head = NULL; + (metadata + i)->bloom = NULL; + (metadata + i)->prev = NULL; + (metadata + i)->next = NULL; + (metadata + i)->kvtag_list_head = NULL; + (metadata + i)->all_storage_region_distributed = 0; + + // ========== Restore KV Tags ========== + BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("kvtags", PDC_STRING)); + + if (kvtags_array != NULL && kvtags_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *kvtag_iter = Bent_iterator_init(kvtags_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(kvtag_iter)) { + BULKI *kvtag_entry = Bent_iterator_next_BULKI(kvtag_iter); + + pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); + kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); + + // Extract key + BULKI_Entity *key_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("key", PDC_STRING)); + int key_len = strlen((char *)key_ent->data) + 1; + kvtag_list->kvtag->name = PDC_malloc(key_len); + memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); + + // Extract size + BULKI_Entity *size_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("size", PDC_STRING)); + memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); + + // Extract type + BULKI_Entity *type_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("type", PDC_STRING)); + memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); + + // Extract value + BULKI_Entity *value_ent = BULKI_get(kvtag_entry, + BULKI_singleton_ENTITY("value", PDC_STRING)); + kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); + memcpy(kvtag_list->kvtag->value, value_ent->data, kvtag_list->kvtag->size); + + DL_APPEND((metadata + i)->kvtag_list_head, kvtag_list); + } } - region_list->region_hist->range = - (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); - region_list->region_hist->bin = - (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); - - if (fread(region_list->region_hist->range, sizeof(double), - region_list->region_hist->nbin * 2, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->range\n"); - } - if (fread(region_list->region_hist->bin, sizeof(uint64_t), region_list->region_hist->nbin, - file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->bin\n"); + // ========== Restore Storage Regions ========== + BULKI_Entity *regions_array = BULKI_get(metadata_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + int n_region = 0; + if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { + n_region = regions_array->count; + BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(region_iter)) { + BULKI *region_entry = Bent_iterator_next_BULKI(region_iter); + + region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); + + // Extract region structure + BULKI_Entity *region_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("region", PDC_STRING)); + memcpy(region_list, region_ent->data, sizeof(region_list_t)); + + // Extract histogram flag + BULKI_Entity *has_hist_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("has_hist", PDC_STRING)); + int has_hist; + memcpy(&has_hist, has_hist_ent->data, sizeof(int)); + + if (has_hist == 1) { + BULKI_Entity *histogram_ent = BULKI_get(region_entry, + BULKI_singleton_ENTITY("histogram", PDC_STRING)); + + if (histogram_ent != NULL && histogram_ent->pdc_type == PDC_BULKI) { + BULKI *histogram = (BULKI *)histogram_ent->data; + + region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); + + BULKI_Entity *dtype_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("dtype", PDC_STRING)); + memcpy(®ion_list->region_hist->dtype, dtype_ent->data, sizeof(int)); + + BULKI_Entity *nbin_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("nbin", PDC_STRING)); + memcpy(®ion_list->region_hist->nbin, nbin_ent->data, sizeof(int)); + + if (region_list->region_hist->nbin == 0) { + LOG_ERROR("Checkpoint file histogram size is 0\n"); + } + + BULKI_Entity *range_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("range", PDC_STRING)); + region_list->region_hist->range = + (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); + memcpy(region_list->region_hist->range, range_ent->data, + sizeof(double) * region_list->region_hist->nbin * 2); + + BULKI_Entity *bin_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("bin", PDC_STRING)); + region_list->region_hist->bin = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); + memcpy(region_list->region_hist->bin, bin_ent->data, + sizeof(uint64_t) * region_list->region_hist->nbin); + + BULKI_Entity *incr_ent = BULKI_get(histogram, + BULKI_singleton_ENTITY("incr", PDC_STRING)); + memcpy(®ion_list->region_hist->incr, incr_ent->data, sizeof(double)); + } + } + + // Initialize region_list fields + region_list->buf = NULL; + region_list->data_size = 1; + for (idx = 0; idx < region_list->ndim; idx++) + region_list->data_size *= region_list->count[idx]; + region_list->is_data_ready = 0; + region_list->shm_fd = 0; + region_list->meta = (metadata + i); + region_list->prev = NULL; + region_list->next = NULL; + region_list->overlap_storage_regions = NULL; + region_list->n_overlap_storage_region = 0; + hg_atomic_init32(&(region_list->buf_map_refcount), 0); + region_list->reg_dirty_from_buf = 0; + region_list->access_type = PDC_NA; + region_list->bulk_handle = NULL; + region_list->lock_handle = NULL; + region_list->addr = NULL; + region_list->obj_id = (metadata + i)->obj_id; + region_list->reg_id = 0; + region_list->from_obj_id = 0; + region_list->client_id = 0; + region_list->is_io_done = 0; + region_list->is_shm_closed = 0; + region_list->seq_id = 0; + region_list->sent_to_server = 0; + region_list->io_cache_region = NULL; + + memset(region_list->shm_addr, 0, ADDR_MAX); + memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); + + if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { + region_list->data_loc_type = PDC_LUSTRE; + } + + DL_APPEND((metadata + i)->storage_region_list_head, region_list); + } } - if (fread(®ion_list->region_hist->incr, sizeof(double), 1, file) != 1) { - LOG_ERROR("Read failed for region_list->region_hist->incr\n"); - } - } - region_list->buf = NULL; - region_list->data_size = 1; - for (idx = 0; idx < region_list->ndim; idx++) - region_list->data_size *= region_list->count[idx]; - region_list->is_data_ready = 0; - region_list->shm_fd = 0; - region_list->meta = (metadata + i); - region_list->prev = NULL; - region_list->next = NULL; - region_list->overlap_storage_regions = NULL; - region_list->n_overlap_storage_region = 0; - hg_atomic_init32(&(region_list->buf_map_refcount), 0); - region_list->reg_dirty_from_buf = 0; - region_list->access_type = PDC_NA; - region_list->bulk_handle = NULL; - region_list->lock_handle = NULL; - region_list->addr = NULL; - region_list->obj_id = (metadata + i)->obj_id; - region_list->reg_id = 0; - region_list->from_obj_id = 0; - region_list->client_id = 0; - region_list->is_io_done = 0; - region_list->is_shm_closed = 0; - region_list->seq_id = 0; - region_list->sent_to_server = 0; - region_list->io_cache_region = NULL; - - memset(region_list->shm_addr, 0, ADDR_MAX); - memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); - - if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { - region_list->data_loc_type = PDC_LUSTRE; - } - - DL_APPEND((metadata + i)->storage_region_list_head, region_list); - } // For j - total_region += n_region; + total_region += n_region; + DL_SORT((metadata + i)->storage_region_list_head, region_cmp); - DL_SORT((metadata + i)->storage_region_list_head, region_cmp); - } // For i + i++; + } + } - nobj += count; - total_mem_usage_g += sizeof(pdc_hash_table_entry_head); - total_mem_usage_g += (sizeof(pdc_metadata_t) * count); + nobj += count; + total_mem_usage_g += sizeof(pdc_hash_table_entry_head); + total_mem_usage_g += (sizeof(pdc_metadata_t) * count); - entry->metadata = NULL; + entry->metadata = NULL; - // Insert the previously read metadata to the linked list (hash table entry) - for (i = 0; i < count; i++) { - elt = metadata + i; - // Add to hash list and bloom filter - ret_value = PDC_Server_hash_table_list_insert(entry, elt); - if (ret_value != SUCCEED) - PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); + // Insert metadata to hash table + for (i = 0; i < count; i++) { + elt = metadata + i; + ret_value = PDC_Server_hash_table_list_insert(entry, elt); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); + } } - n_entry--; - } - - if (fread(&n_objs, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_objs\n"); } - for (i = 0; i < n_objs; ++i) { - data_server_region_t *new_obj_reg = - (data_server_region_t *)PDC_calloc(1, sizeof(struct data_server_region_t)); - new_obj_reg->fd = -1; - new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); - if (fread(&new_obj_reg->obj_id, sizeof(uint64_t), 1, file) != 1) { - LOG_ERROR("Read failed for obj_id\n"); - } - if (fread(&n_region, sizeof(int), 1, file) != 1) { - LOG_ERROR("Read failed for n_region\n"); - } - DL_APPEND(dataserver_region_g, new_obj_reg); - for (j = 0; j < n_region; j++) { - region_list_t *new_region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - if (fread(new_region_list, sizeof(region_list_t), 1, file) != 1) { - LOG_ERROR("Read failed for new_region_list\n"); + // ========== Section 3: Restore Data Server Regions ========== + dataserver_regions_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); + + if (dataserver_regions_array != NULL && dataserver_regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *ds_iter = Bent_iterator_init(dataserver_regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(ds_iter)) { + BULKI *dataserver_obj = Bent_iterator_next_BULKI(ds_iter); + + data_server_region_t *new_obj_reg = + (data_server_region_t *)PDC_calloc(1, sizeof(struct data_server_region_t)); + new_obj_reg->fd = -1; + new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); + + // Extract obj_id + BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, + BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); + + // Extract regions + BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + if (ds_regions_array != NULL && ds_regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *ds_region_iter = Bent_iterator_init(ds_regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(ds_region_iter)) { + BULKI *ds_region = Bent_iterator_next_BULKI(ds_region_iter); + + region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); + + BULKI_Entity *region_ent = BULKI_get(ds_region, + BULKI_singleton_ENTITY("region", PDC_STRING)); + memcpy(region_list, region_ent->data, sizeof(region_list_t)); + + // Initialize fields (similar to above) + region_list->buf = NULL; + region_list->data_size = 1; + for (idx = 0; idx < region_list->ndim; idx++) + region_list->data_size *= region_list->count[idx]; + region_list->is_data_ready = 0; + region_list->shm_fd = 0; + region_list->meta = NULL; + region_list->prev = NULL; + region_list->next = NULL; + region_list->overlap_storage_regions = NULL; + region_list->n_overlap_storage_region = 0; + hg_atomic_init32(&(region_list->buf_map_refcount), 0); + region_list->reg_dirty_from_buf = 0; + region_list->access_type = PDC_NA; + + DL_APPEND(new_obj_reg->region_storage_head, region_list); + } } - DL_APPEND(new_obj_reg->region_storage_head, new_region_list); + + DL_APPEND(dataserver_region_g, new_obj_reg); } } - if (fread(&checkpoint_size, sizeof(uint64_t), 1, file) != 1) { - LOG_ERROR("Read failed for checkpoint size\n"); - } - checkpoint_buf = (char *)PDC_malloc(checkpoint_size); - if (fread(checkpoint_buf, checkpoint_size, 1, file) != 1) { - LOG_ERROR("Read failed for checkpoint buf\n"); + // ========== Section 4: Restore Transfer Query ========== + BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); + if (transfer_query_ent != NULL) { + // Create a buffer copy from the BULKI entity data + char *checkpoint_buf = (char *)PDC_malloc(transfer_query_ent->count); + memcpy(checkpoint_buf, transfer_query_ent->data, transfer_query_ent->count); + + // Call the init function with the checkpoint buffer + transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); + + // Free the buffer + checkpoint_buf = (char *)PDC_free(checkpoint_buf); } - transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); - checkpoint_buf = (char *)PDC_free(checkpoint_buf); - fclose(file); - file = NULL; + // Clean up + BULKI_free(checkpoint_bulki, 1); + file_buffer = PDC_free(file_buffer); #ifdef ENABLE_MPI MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&total_region, &all_n_region, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else - all_nobj = nobj; - all_n_region = total_region; + all_nobj = nobj; + all_n_region = total_region; +#endif + +#ifdef PDC_TIMING + if (pdc_server_rank_g == 0) + LOG_ERROR("Total restart time = %.6f\n", MPI_Wtime() - start); #endif if (pdc_server_rank_g == 0) { - LOG_INFO("Server restarted from saved session, " - "successfully loaded %d containers, %d objects, %d regions...\n", + LOG_INFO("Recovered %d containers, %d objects, %d regions from checkpoint file \n", all_cont, all_nobj, all_n_region); } done: -#ifdef PDC_TIMING - pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; -#endif - FUNC_LEAVE(ret_value); } From 0a53f026ceb0f4a14c4cc0a3800511dfe30d760f Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 27 Feb 2026 15:32:59 -0500 Subject: [PATCH 02/30] Add code to test metadata checkpointing Added code to add kv-tags at scale. Also added code to get and verify tags at scale. The later test code will be used to verify the metadata checkpointing after a server restart. --- src/tests/CMakeLists.txt | 2 + src/tests/tags/kvtag_add_scale.c | 224 ++++++++++++++++++ src/tests/tags/kvtag_get_verify_scale.c | 289 ++++++++++++++++++++++++ 3 files changed, 515 insertions(+) create mode 100644 src/tests/tags/kvtag_add_scale.c create mode 100644 src/tests/tags/kvtag_get_verify_scale.c diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6f0482fdc..574d84d9f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -136,6 +136,8 @@ if(BUILD_MPI_TESTING) tags/kvtag_affix_query_scale tags/kvtag_add_get_benchmark tags/kvtag_add_get_scale + tags/kvtag_add_scale + tags/kvtag_get_verify_scale ) if(PDC_ENABLE_IDIOMS) diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c new file mode 100644 index 000000000..2814a9fbe --- /dev/null +++ b/src/tests/tags/kvtag_add_scale.c @@ -0,0 +1,224 @@ +/* +* Copyright Notice for +* Proactive Data Containers (PDC) Software Library and Utilities +* ----------------------------------------------------------------------------- + +*** Copyright Notice *** + +* Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the +* University of California, through Lawrence Berkeley National Laboratory, +* UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF +* Group (subject to receipt of any required approvals from the U.S. Dept. of +* Energy). All rights reserved. + +* If you have questions about your rights to use or distribute this software, +* please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + +* NOTICE. This Software was developed under funding from the U.S. Department of +* Energy and the U.S. Government consequently retains certain rights. As such, the +* U.S. Government has been granted for itself and others acting on its behalf a +* paid-up, nonexclusive, irrevocable, worldwide license in the Software to +* reproduce, distribute copies to the public, prepare derivative works, and +* perform publicly and display publicly, and to permit other to do so. +*/ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + LOG_INFO("assign_work_to_rank(): Error with input\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + /* Modified: Changed usage to only require n_obj and n_add_tag */ + LOG_JUST_PRINT("%s n_obj n_add_tag\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont, obj_prop; + pdcid_t * obj_ids; + int n_obj, n_add_tag, my_obj, my_obj_s, my_add_tag, my_add_tag_s; + int obj_1percent = 0, tag_1percent = 0; + int proc_num, my_rank, i, v; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + int ret_value = SUCCEED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + /* Modified: Changed argc check from 4 to 3 (removed n_query parameter) */ + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + PGOTO_DONE(FAIL); + } + n_obj = atoi(argv[1]); + n_add_tag = atoi(argv[2]); + /* Removed: n_query assignment */ + + if (n_add_tag > n_obj) { + if (my_rank == 0) + LOG_ERROR("n_add_tag larger than n_obj! Exiting...\n"); + PGOTO_DONE(FAIL); + } + + assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); + /* Removed: assign_work_to_rank call for n_query */ + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + obj_1percent = my_obj / 100; + tag_1percent = my_add_tag / 100; + /* Removed: query_1percent calculation */ + + if (my_rank == 0) + /* Modified: Removed n_query from log message */ + LOG_INFO("Create %d obj, %d tags\n", my_obj, my_add_tag); + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create container property"); + + // create a container + cont = PDCcont_create("c1", cont_prop); + if (cont <= 0) + PGOTO_ERROR(FAIL, "Failed to create container"); + + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create object property"); + + // Create a number of objects, add at least one tag to that object + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj_ids[i] <= 0) + PGOTO_ERROR(FAIL, "Failed to create object"); + + if (i > 0 && i % obj_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / obj_1percent; + int estimated_current_object_number = n_obj / 100 * current_percentage; + double tps = estimated_current_object_number / percent_time; + LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_object_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + + if (my_rank == 0) + LOG_INFO("Total time to create %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, + n_obj / total_time); + + // Add tags + kvtag.name = "Group"; + kvtag.value = (void *)&v; + kvtag.type = PDC_INT; + kvtag.size = sizeof(int); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_add_tag; i++) { + v = i + my_add_tag_s; + if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) + PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); + + if (i % tag_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / tag_1percent; + int estimated_current_tag_number = n_obj / 100 * current_percentage; + double tps = estimated_current_tag_number / percent_time; + LOG_INFO("[TAG PROGRESS %3d%% ] %11d tags, %7.2f seconds, TPS: %10.2f \n", current_percentage, + estimated_current_tag_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, + total_time, n_add_tag / total_time); + + /* Removed: All query-related code (values allocation, get_tag loop, verification loop) */ + + /* Added: Free obj_ids array before cleanup */ + free(obj_ids); + + if (my_rank == 0) { + /* Modified: Changed message to reflect tag addition completion */ + LOG_INFO("Done adding tags\n"); + } + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return ret_value; +} \ No newline at end of file diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c new file mode 100644 index 000000000..60a79b10d --- /dev/null +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -0,0 +1,289 @@ +/* +* Copyright Notice for +* Proactive Data Containers (PDC) Software Library and Utilities +* ----------------------------------------------------------------------------- + +*** Copyright Notice *** + +* Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the +* University of California, through Lawrence Berkeley National Laboratory, +* UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF +* Group (subject to receipt of any required approvals from the U.S. Dept. of +* Energy). All rights reserved. + +* If you have questions about your rights to use or distribute this software, +* please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + +* NOTICE. This Software was developed under funding from the U.S. Department of +* Energy and the U.S. Government consequently retains certain rights. As such, the +* U.S. Government has been granted for itself and others acting on its behalf a +* paid-up, nonexclusive, irrevocable, worldwide license in the Software to +* reproduce, distribute copies to the public, prepare derivative works, and +* perform publicly and display publicly, and to permit other to do so. +*/ + +#include +#include +#include +#include +#include +#include "pdc.h" +#include "pdc_client_connect.h" + +int +assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) +{ + if (rank > size || my_count == NULL || my_start == NULL) { + LOG_INFO("assign_work_to_rank(): Error with input\n"); + return -1; + } + if (nwork < size) { + if (rank < nwork) + *my_count = 1; + else + *my_count = 0; + (*my_start) = rank * (*my_count); + } + else { + (*my_count) = nwork / size; + (*my_start) = rank * (*my_count); + + // Last few ranks may have extra work + if (rank >= size - nwork % size) { + (*my_count)++; + (*my_start) += (rank - (size - nwork % size)); + } + } + + return 1; +} + +void +print_usage(char *name) +{ + /* Modified: Changed usage to only require n_obj and n_query */ + LOG_JUST_PRINT("%s n_obj n_query\n", name); +} + +int +main(int argc, char *argv[]) +{ + pdcid_t pdc, cont_prop, cont; + pdcid_t * obj_ids; + /* Modified: Removed n_add_tag variable, kept n_query */ + int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; + /* Removed: obj_1percent and tag_1percent */ + int query_1percent = 0; + int proc_num, my_rank, i; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + void ** values; + pdc_var_type_t value_type; + size_t value_size; + int ret_value = SUCCEED; + /* Added: Counters for verification statistics */ + int verified_success = 0; + int verified_fail = 0; + int all_verified_success = 0; + int all_verified_fail = 0; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &proc_num); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#endif + /* Modified: Changed argc check from 4 to 3 (removed n_add_tag parameter) */ + if (argc < 3) { + if (my_rank == 0) + print_usage(argv[0]); + PGOTO_DONE(FAIL); + } + n_obj = atoi(argv[1]); + /* Removed: n_add_tag assignment */ + n_query = atoi(argv[2]); + + if (n_query > n_obj) { + if (my_rank == 0) + LOG_ERROR("n_query larger than n_obj! Exiting...\n"); + PGOTO_DONE(FAIL); + } + + /* Removed: assign_work_to_rank call for n_add_tag */ + assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); + assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); + + /* Removed: obj_1percent and tag_1percent calculations */ + query_1percent = my_query / 100; + + if (my_rank == 0) + /* Modified: Log message shows only query count */ + LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); + + // create a pdc + pdc = PDCinit("pdc"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop <= 0) + PGOTO_ERROR(FAIL, "Failed to create container property"); + + /* Added: Open existing container instead of creating new one */ + // Open the existing container + cont = PDCcont_open("c1", pdc); + if (cont <= 0) + PGOTO_ERROR(FAIL, "Failed to open container"); + + /* Removed: object property creation - not needed for opening existing objects */ + + /* Added: Open existing objects instead of creating new ones */ + // Open existing objects + obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + + /* Modified: Changed from PDCobj_create to PDCobj_open */ + for (i = 0; i < my_obj; i++) { + sprintf(obj_name, "obj%d", my_obj_s + i); + obj_ids[i] = PDCobj_open(obj_name, pdc); + if (obj_ids[i] <= 0) + PGOTO_ERROR(FAIL, "Failed to open object"); + + /* Added: Progress reporting for object opening (reusing original format) */ + if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_1percent; + int estimated_current_object_number = n_obj / 100 * current_percentage; + double tps = estimated_current_object_number / percent_time; + LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_object_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + + if (my_rank == 0) + /* Modified: Changed message from "create" to "open" */ + LOG_INFO("Total time to open %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, + n_obj / total_time); + + /* Removed: All tag addition code (kvtag initialization and put_tag loop) */ + + /* Added: Initialize kvtag for query operations */ + // Setup kvtag for queries + kvtag.name = "Group"; + + values = (void **)calloc(my_query, sizeof(void *)); + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + stime = MPI_Wtime(); +#endif + for (i = 0; i < my_query; i++) { + if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, + (void *)&value_size) < 0) + PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + my_query_s); + + if (i % query_1percent == 0) { +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + percent_time = MPI_Wtime() - stime; + if (my_rank == 0) { + int current_percentage = i / query_1percent; + int estimated_current_query_number = n_obj / 100 * current_percentage; + double tps = estimated_current_query_number / percent_time; + LOG_INFO("[QRY PROGRESS %3d%% ] %11d queries, %7.2f seconds, TPS: %10.2f \n", + current_percentage, estimated_current_query_number, percent_time, tps); + } +#endif + } + } + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + total_time, n_query / total_time); + + /* Modified: Changed expected value calculation and added verification counting */ + // The first program adds tags with value = i + my_add_tag_s + // Since we're querying the first my_query objects, and assuming they were tagged + // by ranks in order, we need to calculate the expected value + for (i = 0; i < my_query; i++) { + /* Modified: Calculate expected value based on how tags were originally assigned */ + // When tags were added, each rank tagged objects starting from my_add_tag_s + // For verification, we need to determine which rank added the tag and its offset + int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 + + /* Added: Count successful and failed verifications instead of immediate error */ + if (*(int *)(values[i]) == expected_value) { + verified_success++; + } else { + verified_fail++; + /* Optional: Log first few failures for debugging */ + if (verified_fail <= 10) { + LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", + i + my_query_s, expected_value, *(int *)(values[i])); + } + } + free(values[i]); + } + + free(values); + /* Added: Free obj_ids array */ + free(obj_ids); + + /* Added: Aggregate verification statistics across all ranks */ +#ifdef ENABLE_MPI + MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); +#else + all_verified_success = verified_success; + all_verified_fail = verified_fail; +#endif + + if (my_rank == 0) { + /* Modified: Enhanced completion message with verification statistics */ + LOG_INFO("==============================================================\n"); + LOG_INFO("Verification Summary:\n"); + LOG_INFO(" Total queries: %11d\n", n_query); + LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", + all_verified_success, + 100.0 * all_verified_success / n_query); + LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", + all_verified_fail, + 100.0 * all_verified_fail / n_query); + LOG_INFO("==============================================================\n"); + + if (all_verified_fail > 0) { + LOG_ERROR("WARNING: %d objects failed verification!\n", all_verified_fail); + } else { + LOG_INFO("SUCCESS: All objects verified correctly!\n"); + } + } + + /* Added: Set return value based on verification results */ + if (verified_fail > 0) { + ret_value = FAIL; + } + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + + return ret_value; +} \ No newline at end of file From 5430bebf1ccfae0af0b2f5b159d22c0435cce35a Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 27 Feb 2026 15:54:17 -0500 Subject: [PATCH 03/30] Refactor kvtag add/get/verify test code --- src/tests/tags/kvtag_add_scale.c | 14 ++------ src/tests/tags/kvtag_get_verify_scale.c | 45 +++++++------------------ 2 files changed, 16 insertions(+), 43 deletions(-) diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 2814a9fbe..4e00097f6 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -61,7 +61,7 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) void print_usage(char *name) { - /* Modified: Changed usage to only require n_obj and n_add_tag */ + // required parameters: n_obj and n_add_tag LOG_JUST_PRINT("%s n_obj n_add_tag\n", name); } @@ -83,7 +83,6 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - /* Modified: Changed argc check from 4 to 3 (removed n_query parameter) */ if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); @@ -91,7 +90,6 @@ main(int argc, char *argv[]) } n_obj = atoi(argv[1]); n_add_tag = atoi(argv[2]); - /* Removed: n_query assignment */ if (n_add_tag > n_obj) { if (my_rank == 0) @@ -100,15 +98,12 @@ main(int argc, char *argv[]) } assign_work_to_rank(my_rank, proc_num, n_add_tag, &my_add_tag, &my_add_tag_s); - /* Removed: assign_work_to_rank call for n_query */ assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); obj_1percent = my_obj / 100; tag_1percent = my_add_tag / 100; - /* Removed: query_1percent calculation */ if (my_rank == 0) - /* Modified: Removed n_query from log message */ LOG_INFO("Create %d obj, %d tags\n", my_obj, my_add_tag); // create a pdc @@ -129,7 +124,7 @@ main(int argc, char *argv[]) if (obj_prop <= 0) PGOTO_ERROR(FAIL, "Failed to create object property"); - // Create a number of objects, add at least one tag to that object + // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); #ifdef ENABLE_MPI @@ -205,13 +200,10 @@ main(int argc, char *argv[]) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, total_time, n_add_tag / total_time); - /* Removed: All query-related code (values allocation, get_tag loop, verification loop) */ - - /* Added: Free obj_ids array before cleanup */ free(obj_ids); if (my_rank == 0) { - /* Modified: Changed message to reflect tag addition completion */ + // confirming tag addition completion LOG_INFO("Done adding tags\n"); } diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index 60a79b10d..bdce6df81 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -61,7 +61,7 @@ assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) void print_usage(char *name) { - /* Modified: Changed usage to only require n_obj and n_query */ + // required parameters: n_obj and n_query LOG_JUST_PRINT("%s n_obj n_query\n", name); } @@ -70,9 +70,7 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont; pdcid_t * obj_ids; - /* Modified: Removed n_add_tag variable, kept n_query */ int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; - /* Removed: obj_1percent and tag_1percent */ int query_1percent = 0; int proc_num, my_rank, i; char obj_name[128]; @@ -82,7 +80,7 @@ main(int argc, char *argv[]) pdc_var_type_t value_type; size_t value_size; int ret_value = SUCCEED; - /* Added: Counters for verification statistics */ + // counters for verification statistics int verified_success = 0; int verified_fail = 0; int all_verified_success = 0; @@ -93,14 +91,12 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #endif - /* Modified: Changed argc check from 4 to 3 (removed n_add_tag parameter) */ if (argc < 3) { if (my_rank == 0) print_usage(argv[0]); PGOTO_DONE(FAIL); } n_obj = atoi(argv[1]); - /* Removed: n_add_tag assignment */ n_query = atoi(argv[2]); if (n_query > n_obj) { @@ -109,15 +105,12 @@ main(int argc, char *argv[]) PGOTO_DONE(FAIL); } - /* Removed: assign_work_to_rank call for n_add_tag */ assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); - /* Removed: obj_1percent and tag_1percent calculations */ query_1percent = my_query / 100; if (my_rank == 0) - /* Modified: Log message shows only query count */ LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); // create a pdc @@ -128,15 +121,11 @@ main(int argc, char *argv[]) if (cont_prop <= 0) PGOTO_ERROR(FAIL, "Failed to create container property"); - /* Added: Open existing container instead of creating new one */ // Open the existing container cont = PDCcont_open("c1", pdc); if (cont <= 0) PGOTO_ERROR(FAIL, "Failed to open container"); - /* Removed: object property creation - not needed for opening existing objects */ - - /* Added: Open existing objects instead of creating new ones */ // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); @@ -145,14 +134,14 @@ main(int argc, char *argv[]) stime = MPI_Wtime(); #endif - /* Modified: Changed from PDCobj_create to PDCobj_open */ + // open already created PDC objects and query tags for (i = 0; i < my_obj; i++) { sprintf(obj_name, "obj%d", my_obj_s + i); obj_ids[i] = PDCobj_open(obj_name, pdc); if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to open object"); - /* Added: Progress reporting for object opening (reusing original format) */ + // progress reporting for object opening if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); @@ -174,13 +163,9 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) - /* Modified: Changed message from "create" to "open" */ LOG_INFO("Total time to open %11d objects: %7.2f , throughput %10.2f \n", n_obj, total_time, n_obj / total_time); - /* Removed: All tag addition code (kvtag initialization and put_tag loop) */ - - /* Added: Initialize kvtag for query operations */ // Setup kvtag for queries kvtag.name = "Group"; @@ -218,22 +203,19 @@ main(int argc, char *argv[]) LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, total_time, n_query / total_time); - /* Modified: Changed expected value calculation and added verification counting */ - // The first program adds tags with value = i + my_add_tag_s - // Since we're querying the first my_query objects, and assuming they were tagged - // by ranks in order, we need to calculate the expected value + // `kvtag_add_get_scale' program adds tags with value = i + my_add_tag_s + // calculate and checking the expected value for (i = 0; i < my_query; i++) { - /* Modified: Calculate expected value based on how tags were originally assigned */ - // When tags were added, each rank tagged objects starting from my_add_tag_s - // For verification, we need to determine which rank added the tag and its offset + // when tags were added, each rank tagged objects starting from my_add_tag_s + // for verification, we need to determine which rank added the tag and its offset int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 - /* Added: Count successful and failed verifications instead of immediate error */ + // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { verified_success++; } else { verified_fail++; - /* Optional: Log first few failures for debugging */ + // log first 10 failures for debugging if (verified_fail <= 10) { LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", i + my_query_s, expected_value, *(int *)(values[i])); @@ -243,10 +225,9 @@ main(int argc, char *argv[]) } free(values); - /* Added: Free obj_ids array */ free(obj_ids); - /* Added: Aggregate verification statistics across all ranks */ + // aggregate verification statistics across all ranks #ifdef ENABLE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -256,7 +237,7 @@ main(int argc, char *argv[]) #endif if (my_rank == 0) { - /* Modified: Enhanced completion message with verification statistics */ + // completion message with verification statistics LOG_INFO("==============================================================\n"); LOG_INFO("Verification Summary:\n"); LOG_INFO(" Total queries: %11d\n", n_query); @@ -275,7 +256,7 @@ main(int argc, char *argv[]) } } - /* Added: Set return value based on verification results */ + // set return value based on verification results if (verified_fail > 0) { ret_value = FAIL; } From 602c5e66734a81a94e395b19e5756d05c1db2f7f Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:03:40 -0400 Subject: [PATCH 04/30] Add bulki-based new transfer request metadata query for checkpointing --- src/server/pdc_server.c | 43 ++-- ...dc_server_region_transfer_metadata_query.h | 5 + ...dc_server_region_transfer_metadata_query.c | 202 ++++++++++++++++++ 3 files changed, 235 insertions(+), 15 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 9b0ae94c1..4038e7217 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -949,7 +949,7 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) } else { // We are starting a brand new server - transfer_request_metadata_query_init(pdc_server_size_g, NULL); + transfer_request_metadata_query_init_bulki(pdc_server_size_g, NULL); if (is_hash_table_init_g != 1) { ret_value = PDC_Server_init_hash_table(); if (ret_value != SUCCEED) @@ -1454,14 +1454,18 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), dataserver_regions_array); - // ========== Section 4: Transfer Request Metadata Query ========== - char * query_checkpoint; - uint64_t query_checkpoint_size; - transfer_request_metadata_query_checkpoint(&query_checkpoint, &query_checkpoint_size); + // transfer request metadata query + BULKI *transfer_query_bulki = NULL; + ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); + + if (ret_value != SUCCEED || transfer_query_bulki == NULL) { + LOG_ERROR("Failed to create transfer query checkpoint\n"); + PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); + } BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), - BULKI_ENTITY(query_checkpoint, query_checkpoint_size, PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); @@ -1940,19 +1944,28 @@ PDC_Server_restart(char *filename) } } - // ========== Section 4: Restore Transfer Query ========== + // restore transfer query BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); - if (transfer_query_ent != NULL) { - // Create a buffer copy from the BULKI entity data - char *checkpoint_buf = (char *)PDC_malloc(transfer_query_ent->count); - memcpy(checkpoint_buf, transfer_query_ent->data, transfer_query_ent->count); - // Call the init function with the checkpoint buffer - transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); + if (transfer_query_ent != NULL && transfer_query_ent->pdc_type == PDC_BULKI) { + // Extract the nested BULKI containing transfer query data + BULKI *transfer_query_bulki = (BULKI *)transfer_query_ent->data; - // Free the buffer - checkpoint_buf = (char *)PDC_free(checkpoint_buf); + // initialize transfer query system with BULKI checkpoint + ret_value = transfer_request_metadata_query_init_bulki(pdc_server_size_g, transfer_query_bulki); + if (ret_value != SUCCEED) { + LOG_ERROR("Failed to restore transfer query from checkpoint\n"); + PGOTO_ERROR(FAIL, "Transfer query restoration failed"); + } + } + else { + // note: no transfer query data in checkpoint, initialize fresh + ret_value = transfer_request_metadata_query_init_bulki(pdc_server_size_g, NULL); + if (ret_value != SUCCEED) { + LOG_ERROR("Failed to initialize transfer query system\n"); + PGOTO_ERROR(FAIL, "Transfer query initialization failed"); + } } // Clean up diff --git a/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h b/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h index fc2ec3a7a..5755d5e4e 100644 --- a/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h +++ b/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h @@ -1,9 +1,14 @@ #ifndef PDC_SERVER_REGION_TRANSFER_METADATA_QUERY_H #define PDC_SERVER_REGION_TRANSFER_METADATA_QUERY_H +#include "bulki.h" +#include "bulki_serde.h" + perr_t transfer_request_metadata_query_init(int pdc_server_size_input, char *checkpoint); +perr_t transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *checkpoint_bulki); perr_t transfer_request_metadata_query_finalize(); perr_t transfer_request_metadata_query_checkpoint(char **checkpoint, uint64_t *checkpoint_size); +perr_t transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki); perr_t transfer_request_metadata_query_lookup_query_buf(uint64_t query_id, char **buf_ptr); uint64_t transfer_request_metadata_query_parse(int32_t n_objs, char *buf, uint8_t partition_type, uint64_t *total_buf_size_ptr); diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index c66ecd905..8a2c8dc8e 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -53,6 +53,128 @@ static uint64_t transfer_request_metadata_query_append(uint64_t obj_id, int ndim static uint64_t metadata_query_buf_create(pdc_obj_region_metadata *regions, int size, uint64_t *total_buf_size_ptr); +perr_t +transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *checkpoint_bulki) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + metadata_server_objs = NULL; + metadata_server_objs_end = NULL; + metadata_query_buf_head = NULL; + metadata_query_buf_end = NULL; + pdc_server_size = pdc_server_size_input; + data_server_bytes = (uint64_t *)PDC_calloc(pdc_server_size, sizeof(uint64_t)); + query_id_g = 100000; + + pthread_mutex_init(&metadata_query_mutex, NULL); + + if (checkpoint_bulki != NULL) { + BULKI_Entity *objects_array = BULKI_get(checkpoint_bulki, + BULKI_singleton_ENTITY("objects", PDC_STRING)); + + if (objects_array == NULL || objects_array->pdc_type != PDC_BULKI) { + LOG_ERROR("Invalid transfer query checkpoint: missing or invalid 'objects' field\n"); + PGOTO_ERROR(FAIL, "Invalid checkpoint format"); + } + + BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(objects_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(obj_iter)) { + BULKI *obj_bulki = Bent_iterator_next_BULKI(obj_iter); + + pdc_obj_metadata_pkg *obj_pkg = + (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + + BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + if (obj_id_ent == NULL) { + LOG_ERROR("Missing obj_id in checkpoint object\n"); + PDC_free(obj_pkg); + continue; + } + memcpy(&obj_pkg->obj_id, obj_id_ent->data, sizeof(uint64_t)); + + BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("ndim", PDC_STRING)); + if (ndim_ent == NULL) { + LOG_ERROR("Missing ndim in checkpoint object\n"); + PDC_free(obj_pkg); + continue; + } + memcpy(&obj_pkg->ndim, ndim_ent->data, sizeof(int)); + + obj_pkg->regions = NULL; + obj_pkg->regions_end = NULL; + obj_pkg->next = NULL; + + BULKI_Entity *regions_array = BULKI_get(obj_bulki, + BULKI_singleton_ENTITY("regions", PDC_STRING)); + + if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { + BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + + while (Bent_iterator_has_next_BULKI(region_iter)) { + BULKI *region_bulki = Bent_iterator_next_BULKI(region_iter); + + pdc_region_metadata_pkg *region_pkg = + (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); + + region_pkg->reg_offset = (uint64_t *)PDC_malloc(sizeof(uint64_t) * obj_pkg->ndim * 2); + region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; + + BULKI_Entity *server_id_ent = BULKI_get(region_bulki, + BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); + if (server_id_ent != NULL) { + memcpy(®ion_pkg->data_server_id, server_id_ent->data, sizeof(uint32_t)); + } else { + LOG_ERROR("Missing data_server_id in checkpoint region\n"); + PDC_free(region_pkg->reg_offset); + PDC_free(region_pkg); + continue; + } + + BULKI_Entity *offset_size_ent = BULKI_get(region_bulki, + BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); + if (offset_size_ent != NULL) { + memcpy(region_pkg->reg_offset, offset_size_ent->data, + sizeof(uint64_t) * obj_pkg->ndim * 2); + } else { + LOG_ERROR("Missing reg_offset_size in checkpoint region\n"); + PDC_free(region_pkg->reg_offset); + PDC_free(region_pkg); + continue; + } + + region_pkg->next = NULL; + + if (obj_pkg->regions == NULL) { + obj_pkg->regions = region_pkg; + obj_pkg->regions_end = region_pkg; + } else { + obj_pkg->regions_end->next = region_pkg; + obj_pkg->regions_end = region_pkg; + } + } + } + + if (metadata_server_objs == NULL) { + metadata_server_objs = obj_pkg; + metadata_server_objs_end = obj_pkg; + } else { + metadata_server_objs_end->next = obj_pkg; + metadata_server_objs_end = obj_pkg; + } + } + + LOG_DEBUG("Transfer query checkpoint restored successfully\n"); + } + +done: + FUNC_LEAVE(ret_value); +} + /** * Entry function for this class. Should be only called once at the beginning of Server init. * If checkpoint is not NULL, then load previously checkpointed metadata to static variables. @@ -167,6 +289,86 @@ transfer_request_metadata_query_finalize() FUNC_LEAVE(ret_value); } +perr_t +transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + pdc_obj_metadata_pkg * obj_temp; + pdc_region_metadata_pkg *region_temp; + int obj_count = 0; + BULKI * bulki = NULL; + +// if (checkpoint_bulki == NULL) { +// LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); +// PGOTO_ERROR(FAIL, "Invalid parameter"); +// } + + pthread_mutex_lock(&metadata_query_mutex); + + bulki = BULKI_init(1); + BULKI_Entity *objects_array = empty_BULKI_Array_Entity(); + + obj_temp = metadata_server_objs; + while (obj_temp) { + BULKI *obj_bulki = BULKI_init(3); + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("ndim", PDC_STRING), + BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); + + BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); + + region_temp = obj_temp->regions; + while (region_temp) { + BULKI *region_bulki = BULKI_init(2); + + BULKI_put(region_bulki, + BULKI_singleton_ENTITY("data_server_id", PDC_STRING), + BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); + + BULKI_put(region_bulki, + BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, + PDC_UINT64, PDC_CLS_ARRAY)); + + BULKI_ENTITY_append_BULKI(regions_array, region_bulki); + region_temp = region_temp->next; + } + + BULKI_put(obj_bulki, + BULKI_singleton_ENTITY("regions", PDC_STRING), + regions_array); + + BULKI_ENTITY_append_BULKI(objects_array, obj_bulki); + + obj_count++; + obj_temp = obj_temp->next; + } + + BULKI_put(bulki, + BULKI_singleton_ENTITY("objects", PDC_STRING), + objects_array); + + pthread_mutex_unlock(&metadata_query_mutex); + + *checkpoint_bulki = bulki; + + LOG_DEBUG("Transfer query checkpoint created: %d objects\n", obj_count); + +done: + if (ret_value != SUCCEED && bulki != NULL) { + BULKI_free(bulki, 1); + } + + FUNC_LEAVE(ret_value); +} + /** * Checkpoint static variables in this file into a contiguous buffer. * checkpoint_size is the total number of bytes allocated. From 050a723f9e4e71abd09a8f5951f5072a41645bee Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:11:06 -0400 Subject: [PATCH 05/30] Update call to bulki serialization and de-serialization directly to and from file instead of using a buffer Also added version control for the bulki-based checkpointing. --- src/server/pdc_server.c | 85 +++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 4038e7217..eeb365b85 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -77,6 +77,14 @@ sqlite3 *sqlite3_db_g; #include "bulki.h" #include "bulki_serde.h" +// PDC checkpoint version management +#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 +#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 + +#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length +#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 +#define PDC_CHECKPOINT_VERSION_MATCH(m, v) (strncmp((m), (v), 8) == 0) + // Check how long PDC has run every OP_INTERVAL operations #define PDC_CHECKPOINT_CHK_OP_INTERVAL 2000 // Checkpoint every INTERVAL_SEC second and at least OP_INTERVAL operations @@ -1194,16 +1202,12 @@ PDC_Server_checkpoint() HashTablePair pair; char checkpoint_file[ADDR_MAX], checkpoint_file_local[ADDR_MAX], cmd[4096]; HashTableIterator hash_table_iter; -// char * checkpoint; char * env_char; -// uint64_t checkpoint_size; bool use_tmpfs = false; FILE * file; // BULKI-specific variables BULKI *checkpoint_bulki = NULL; - size_t serialized_size = 0; - void * serialized_buffer = NULL; #ifdef PDC_TIMING // Timing @@ -1234,7 +1238,12 @@ PDC_Server_checkpoint() // Initialize BULKI structure - estimate initial capacity checkpoint_bulki = BULKI_init(10); // todo: this should be 4 - // ========== Section 1: Checkpoint Containers ========== + // BULKI version number for validation + BULKI_put(checkpoint_bulki, + BULKI_singleton_ENTITY("version_number", PDC_STRING), + BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + + // checkpoint containers n_entry = hash_table_num_entries(container_hash_table_g); BULKI_Entity *containers_array = empty_BULKI_Array_Entity(); @@ -1468,8 +1477,6 @@ PDC_Server_checkpoint() BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== - serialized_buffer = BULKI_serialize(checkpoint_bulki, &serialized_size); - if (use_tmpfs) file = fopen(checkpoint_file_local, "wb"); else @@ -1478,17 +1485,13 @@ PDC_Server_checkpoint() if (file == NULL) PGOTO_ERROR(FAIL, "Checkpoint file open error"); - // Write BULKI magic header for validation - uint32_t magic = 0x424C4B49; // "BLKI" in hex - fwrite(&magic, sizeof(uint32_t), 1, file); - - // Write serialized data - fwrite(serialized_buffer, serialized_size, 1, file); - fclose(file); + // note: BULKI_Entity_serialize_to_file() still do in-memory serialization + // note: need to implement a streaming file writing without in-memory serialization + BULKI_serialize_to_file(checkpoint_bulki, file); + file = NULL; // file was closed by BULKI_serialize_to_file() // Clean up BULKI_free(checkpoint_bulki, 1); - serialized_buffer = PDC_free(serialized_buffer); if (use_tmpfs) { #ifdef PDC_TIMING @@ -1575,9 +1578,8 @@ PDC_Server_restart(char *filename) BULKI_Entity * containers_array = NULL; BULKI_Entity * metadata_entries_array = NULL; BULKI_Entity * dataserver_regions_array = NULL; - void * file_buffer = NULL; - size_t file_size; - uint32_t magic; + char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; + FILE * file = NULL; #ifdef PDC_TIMING double start = MPI_Wtime(); @@ -1588,38 +1590,39 @@ PDC_Server_restart(char *filename) if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Server_init_hash_table"); - // Read entire file into memory - FILE *file = fopen(filename, "rb"); - if (file == NULL) + // open file + file = fopen(filename, "rb"); + if (file == NULL) { PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); - - fseek(file, 0, SEEK_END); - file_size = ftell(file); - fseek(file, 0, SEEK_SET); - - file_buffer = PDC_malloc(file_size); - if (fread(file_buffer, file_size, 1, file) != 1) { - LOG_ERROR("Failed to read checkpoint file\n"); - fclose(file); - PGOTO_ERROR(FAIL, "Read failed for checkpoint file"); } - fclose(file); - // Validate magic header - memcpy(&magic, file_buffer, sizeof(uint32_t)); - if (magic != 0x424C4B49) { - LOG_ERROR("Invalid checkpoint file magic header\n"); - PGOTO_ERROR(FAIL, "Invalid checkpoint format"); - } + // Read and validate magic header +// fread(magic, 1, PDC_CHECKPOINT_MAGIC_LEN, file); +// magic[PDC_CHECKPOINT_MAGIC_LEN] = '\0'; +// if (!PDC_CHECKPOINT_VERSION_MATCH(magic, PDC_CHECKPOINT_MAGIC_CURRENT)) { +// LOG_ERROR("Invalid checkpoint file magic header: '%s'\n", magic); +// fclose(file); +// PGOTO_ERROR(FAIL, "Invalid checkpoint format"); +// } + + // Note: BULKI_deserialize_from_file will close the file + checkpoint_bulki = BULKI_deserialize_from_file(file); + file = NULL; // File was closed by BULKI_deserialize_from_file - // Deserialize BULKI - checkpoint_bulki = BULKI_deserialize((char *)file_buffer + sizeof(uint32_t)); if (checkpoint_bulki == NULL) { LOG_ERROR("Failed to deserialize checkpoint file\n"); PGOTO_ERROR(FAIL, "Deserialization failed"); } - // ========== Section 1: Restore Containers ========== + BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); + int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + if (!equal) { + LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", + PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); + PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); + } + + // restore containers containers_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING)); From 7efb94527b414f6fdcd49b89c79a68660d219a4d Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:12:24 -0400 Subject: [PATCH 06/30] Refactor bulki-based checkpointing integration in pdc-server --- src/server/pdc_server.c | 87 +++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index eeb365b85..ae9696f6c 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1235,8 +1235,8 @@ PDC_Server_checkpoint() if (pdc_server_rank_g == 0) LOG_INFO("Checkpoint file [%s]\n", checkpoint_file); - // Initialize BULKI structure - estimate initial capacity - checkpoint_bulki = BULKI_init(10); // todo: this should be 4 + // initialize BULKI structure - estimate initial capacity + checkpoint_bulki = BULKI_init(5); // BULKI version number for validation BULKI_put(checkpoint_bulki, @@ -1260,7 +1260,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); - // Store the container structure as binary blob + // store the container structure as binary blob BULKI_put(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); @@ -1272,7 +1272,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); - // ========== Section 2: Checkpoint Metadata Hash Table ========== + // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity(); @@ -1295,20 +1295,20 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); - // Array of metadata objects + // array of metadata objects BULKI_Entity *metadata_objs_array = empty_BULKI_Array_Entity(); - // Iterate every metadata structure in current entry + // iterate every metadata structure in current entry DL_FOREACH(head->metadata, elt) { - BULKI *metadata_obj = BULKI_init(4); // todo: maybe this should be 3 + BULKI *metadata_obj = BULKI_init(3); - // Store metadata structure + // store metadata structure BULKI_put(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); - // ========== KV Tags ========== + // kv tags DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity(); @@ -1316,7 +1316,6 @@ PDC_Server_checkpoint() { BULKI *kvtag_entry = BULKI_init(4); - key_len = strlen(kvlist_elt->kvtag->name) + 1; BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); @@ -1341,7 +1340,7 @@ PDC_Server_checkpoint() BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); - // ========== Storage Regions ========== + // storage regions n_region = 0; DL_COUNT(elt->storage_region_list_head, region_elt, n_region); @@ -1352,12 +1351,12 @@ PDC_Server_checkpoint() { BULKI *region_entry = BULKI_init(3); - // Store region structure + // store region structure BULKI_put(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); - // Store histogram if present + // store histogram if present int has_hist = 0; if (region_elt->region_hist != NULL) has_hist = 1; @@ -1412,20 +1411,20 @@ PDC_Server_checkpoint() metadata_size++; region_count += n_region; - } // End for metadata entry linked list + } // end for metadata entry linked list BULKI_put(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); - } // End for hash table metadata entry + } // end for hash table metadata entry BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), metadata_entries_array); - // ========== Section 3: Data Server Regions ========== + // data server regions data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); @@ -1433,7 +1432,7 @@ PDC_Server_checkpoint() DL_FOREACH(dataserver_region_g, region) { - BULKI *dataserver_obj = BULKI_init(3); + BULKI *dataserver_obj = BULKI_init(2); BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), @@ -1519,8 +1518,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs (BULKI size: %zu bytes)\n", - metadata_size, region_count, checkpoint_time_rank, serialized_size); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", + metadata_size, region_count, checkpoint_time_rank); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1660,7 +1659,7 @@ PDC_Server_restart(char *filename) } } - // ========== Section 2: Restore Metadata ========== + // restore metadata metadata_entries_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); @@ -1670,20 +1669,20 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(entry_iter)) { BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); - // Extract n_obj + // extract n_obj BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING)); int count; memcpy(&count, n_obj_ent->data, sizeof(int)); - // Extract hash key + // extract hash key BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); - // Reconstruct hash table entry + // reconstruct hash table entry entry = (pdc_hash_table_entry_head *)PDC_malloc(sizeof(pdc_hash_table_entry_head)); entry->n_obj = 0; entry->bloom = NULL; @@ -1692,7 +1691,7 @@ PDC_Server_restart(char *filename) metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); - // Extract metadata objects array + // extract metadata objects array BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); @@ -1703,12 +1702,12 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); - // Extract metadata structure + // extract metadata structure BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING)); memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); - // Initialize pointers + // initialize pointers (metadata + i)->storage_region_list_head = NULL; (metadata + i)->region_lock_head = NULL; (metadata + i)->region_map_head = NULL; @@ -1719,7 +1718,7 @@ PDC_Server_restart(char *filename) (metadata + i)->kvtag_list_head = NULL; (metadata + i)->all_storage_region_distributed = 0; - // ========== Restore KV Tags ========== + // restore kv tags BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING)); @@ -1732,24 +1731,24 @@ PDC_Server_restart(char *filename) pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); - // Extract key + // extract key BULKI_Entity *key_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING)); int key_len = strlen((char *)key_ent->data) + 1; kvtag_list->kvtag->name = PDC_malloc(key_len); memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); - // Extract size + // extract size BULKI_Entity *size_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING)); memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); - // Extract type + // extract type BULKI_Entity *type_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING)); memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); - // Extract value + // extract value BULKI_Entity *value_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING)); kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); @@ -1759,7 +1758,7 @@ PDC_Server_restart(char *filename) } } - // ========== Restore Storage Regions ========== + // restore storage regions BULKI_Entity *regions_array = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); @@ -1773,12 +1772,12 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - // Extract region structure + // extract region structure BULKI_Entity *region_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); - // Extract histogram flag + // extract histogram flag BULKI_Entity *has_hist_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING)); int has_hist; @@ -1825,7 +1824,7 @@ PDC_Server_restart(char *filename) } } - // Initialize region_list fields + // initialize region_list fields region_list->buf = NULL; region_list->data_size = 1; for (idx = 0; idx < region_list->ndim; idx++) @@ -1877,7 +1876,7 @@ PDC_Server_restart(char *filename) entry->metadata = NULL; - // Insert metadata to hash table + // insert metadata to hash table for (i = 0; i < count; i++) { elt = metadata + i; ret_value = PDC_Server_hash_table_list_insert(entry, elt); @@ -1887,7 +1886,7 @@ PDC_Server_restart(char *filename) } } - // ========== Section 3: Restore Data Server Regions ========== + // restore data server regions dataserver_regions_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); @@ -1902,12 +1901,12 @@ PDC_Server_restart(char *filename) new_obj_reg->fd = -1; new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); - // Extract obj_id + // extract obj_id BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); - // Extract regions + // extract regions BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); @@ -1923,7 +1922,7 @@ PDC_Server_restart(char *filename) BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); - // Initialize fields (similar to above) + // initialize fields (similar to above) region_list->buf = NULL; region_list->data_size = 1; for (idx = 0; idx < region_list->ndim; idx++) @@ -1971,9 +1970,8 @@ PDC_Server_restart(char *filename) } } - // Clean up + // clean up BULKI_free(checkpoint_bulki, 1); - file_buffer = PDC_free(file_buffer); #ifdef ENABLE_MPI MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -1994,6 +1992,11 @@ PDC_Server_restart(char *filename) } done: + // ensure file is closed if error occurred before BULKI_deserialize_from_file + if (file != NULL) { + fclose(file); + } + FUNC_LEAVE(ret_value); } From 684bf327e625c40f7166c11078581584be97cba6 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 17 Mar 2026 13:44:56 -0400 Subject: [PATCH 07/30] Refactor kvtag-get-verify test code --- src/tests/tags/kvtag_get_verify_scale.c | 67 ++++++++----------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index bdce6df81..e624c95d5 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -62,7 +62,7 @@ void print_usage(char *name) { // required parameters: n_obj and n_query - LOG_JUST_PRINT("%s n_obj n_query\n", name); + LOG_JUST_PRINT("%s n_obj n_tag\n", name); } int @@ -70,8 +70,8 @@ main(int argc, char *argv[]) { pdcid_t pdc, cont_prop, cont; pdcid_t * obj_ids; - int n_obj, n_query, my_obj, my_obj_s, my_query, my_query_s; - int query_1percent = 0; + int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; + int obj_1percent = 0; int proc_num, my_rank, i; char obj_name[128]; double stime, total_time, percent_time; @@ -105,22 +105,17 @@ main(int argc, char *argv[]) PGOTO_DONE(FAIL); } - assign_work_to_rank(my_rank, proc_num, n_query, &my_query, &my_query_s); + assign_work_to_rank(my_rank, proc_num, n_query, &n_tag, &n_tag_s); assign_work_to_rank(my_rank, proc_num, n_obj, &my_obj, &my_obj_s); - query_1percent = my_query / 100; + obj_1percent = my_obj / 100; if (my_rank == 0) - LOG_INFO("Open %d obj, query %d tags\n", my_obj, my_query); + LOG_INFO("Open %d obj, query %d tags\n", my_obj, n_tag); // create a pdc pdc = PDCinit("pdc"); - // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); - if (cont_prop <= 0) - PGOTO_ERROR(FAIL, "Failed to create container property"); - // Open the existing container cont = PDCcont_open("c1", pdc); if (cont <= 0) @@ -142,12 +137,12 @@ main(int argc, char *argv[]) PGOTO_ERROR(FAIL, "Failed to open object"); // progress reporting for object opening - if (i > 0 && query_1percent > 0 && i % query_1percent == 0) { + if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { - int current_percentage = i / query_1percent; + int current_percentage = i / obj_1percent; int estimated_current_object_number = n_obj / 100 * current_percentage; double tps = estimated_current_object_number / percent_time; LOG_INFO("[OBJ PROGRESS %3d%% ] %11d objects, %7.2f seconds, TPS: %10.2f \n", @@ -169,46 +164,18 @@ main(int argc, char *argv[]) // Setup kvtag for queries kvtag.name = "Group"; - values = (void **)calloc(my_query, sizeof(void *)); + values = (void **)calloc(n_tag, sizeof(void *)); #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #endif - for (i = 0; i < my_query; i++) { + for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, (void *)&value_size) < 0) - PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + my_query_s); - - if (i % query_1percent == 0) { -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - percent_time = MPI_Wtime() - stime; - if (my_rank == 0) { - int current_percentage = i / query_1percent; - int estimated_current_query_number = n_obj / 100 * current_percentage; - double tps = estimated_current_query_number / percent_time; - LOG_INFO("[QRY PROGRESS %3d%% ] %11d queries, %7.2f seconds, TPS: %10.2f \n", - current_percentage, estimated_current_query_number, percent_time, tps); - } -#endif - } - } - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); - total_time = MPI_Wtime() - stime; -#endif - if (my_rank == 0) - LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, - total_time, n_query / total_time); + PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + n_tag_s); - // `kvtag_add_get_scale' program adds tags with value = i + my_add_tag_s - // calculate and checking the expected value - for (i = 0; i < my_query; i++) { - // when tags were added, each rank tagged objects starting from my_add_tag_s - // for verification, we need to determine which rank added the tag and its offset - int expected_value = i + my_query_s; // Assuming tags were added in order starting from 0 + int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { @@ -218,12 +185,20 @@ main(int argc, char *argv[]) // log first 10 failures for debugging if (verified_fail <= 10) { LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", - i + my_query_s, expected_value, *(int *)(values[i])); + i + n_tag_s, expected_value, *(int *)(values[i])); } } free(values[i]); } +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + total_time = MPI_Wtime() - stime; +#endif + if (my_rank == 0) + LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + total_time, n_query / total_time); + free(values); free(obj_ids); From 2c92c0b280edd5f3e8c5c01c67b56c9dc516f9e5 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Wed, 18 Mar 2026 00:44:02 -0400 Subject: [PATCH 08/30] Refactor code Added comment, fixed formatting suggested by GitHub clang-format style checker, brought back PDC_TIMING log from the stable version. --- src/server/pdc_server.c | 27 +++++++++---------------- src/tests/tags/kvtag_get_verify_scale.c | 11 +++++----- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index ae9696f6c..1b0d1ec21 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1465,7 +1465,6 @@ PDC_Server_checkpoint() // transfer request metadata query BULKI *transfer_query_bulki = NULL; ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); - if (ret_value != SUCCEED || transfer_query_bulki == NULL) { LOG_ERROR("Failed to create transfer query checkpoint\n"); PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); @@ -1595,16 +1594,7 @@ PDC_Server_restart(char *filename) PGOTO_ERROR(FAIL, "Error with fopen, filename: [%s]", filename); } - // Read and validate magic header -// fread(magic, 1, PDC_CHECKPOINT_MAGIC_LEN, file); -// magic[PDC_CHECKPOINT_MAGIC_LEN] = '\0'; -// if (!PDC_CHECKPOINT_VERSION_MATCH(magic, PDC_CHECKPOINT_MAGIC_CURRENT)) { -// LOG_ERROR("Invalid checkpoint file magic header: '%s'\n", magic); -// fclose(file); -// PGOTO_ERROR(FAIL, "Invalid checkpoint format"); -// } - - // Note: BULKI_deserialize_from_file will close the file + // note: BULKI_deserialize_from_file will close the file checkpoint_bulki = BULKI_deserialize_from_file(file); file = NULL; // File was closed by BULKI_deserialize_from_file @@ -1613,11 +1603,13 @@ PDC_Server_restart(char *filename) PGOTO_ERROR(FAIL, "Deserialization failed"); } + // read and validate bulki version BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); if (!equal) { LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", - PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); + PDC_CHECKPOINT_MAGIC_CURRENT, + version_entity ? (char *)version_entity->data : "NULL"); PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); } @@ -1981,17 +1973,16 @@ PDC_Server_restart(char *filename) all_n_region = total_region; #endif -#ifdef PDC_TIMING - if (pdc_server_rank_g == 0) - LOG_ERROR("Total restart time = %.6f\n", MPI_Wtime() - start); -#endif - if (pdc_server_rank_g == 0) { - LOG_INFO("Recovered %d containers, %d objects, %d regions from checkpoint file \n", + LOG_INFO("Server restarted from saved session, " + "successfully loaded %d containers, %d objects, %d regions...\n", all_cont, all_nobj, all_n_region); } done: +#ifdef PDC_TIMING + pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; +#endif // ensure file is closed if error occurred before BULKI_deserialize_from_file if (file != NULL) { fclose(file); diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index e624c95d5..a7ffd21c1 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -180,7 +180,8 @@ main(int argc, char *argv[]) // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { verified_success++; - } else { + } + else { verified_fail++; // log first 10 failures for debugging if (verified_fail <= 10) { @@ -196,7 +197,7 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - LOG_INFO("Total time to retrieve 1 tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, n_obj, total_time, n_query / total_time); free(values); @@ -216,11 +217,9 @@ main(int argc, char *argv[]) LOG_INFO("==============================================================\n"); LOG_INFO("Verification Summary:\n"); LOG_INFO(" Total queries: %11d\n", n_query); - LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", - all_verified_success, + LOG_INFO(" Successfully verified: %11d (%6.2f%%)\n", all_verified_success, 100.0 * all_verified_success / n_query); - LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", - all_verified_fail, + LOG_INFO(" Failed verification: %11d (%6.2f%%)\n", all_verified_fail, 100.0 * all_verified_fail / n_query); LOG_INFO("==============================================================\n"); From 65c546f56054839f1b3ed8989fa9f4377e5fa987 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 26 Mar 2026 20:59:01 +0000 Subject: [PATCH 09/30] Committing clang-format changes --- src/server/pdc_server.c | 284 ++++++++---------- ...dc_server_region_transfer_metadata_query.c | 85 +++--- src/tests/tags/kvtag_get_verify_scale.c | 41 +-- 3 files changed, 186 insertions(+), 224 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1b0d1ec21..1a43b6ea7 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -78,11 +78,11 @@ sqlite3 *sqlite3_db_g; #include "bulki_serde.h" // PDC checkpoint version management -#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 -#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 +#define PDC_CHECKPOINT_MAGIC_2026 "PDC26.03" // Year 2026, month 03 +#define PDC_CHECKPOINT_MAGIC_2027 "PDC27.01" // Year 2027, month 01 -#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length -#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 +#define PDC_CHECKPOINT_MAGIC_LEN 8 // All must be same length +#define PDC_CHECKPOINT_MAGIC_CURRENT PDC_CHECKPOINT_MAGIC_2026 #define PDC_CHECKPOINT_VERSION_MATCH(m, v) (strncmp((m), (v), 8) == 0) // Check how long PDC has run every OP_INTERVAL operations @@ -1239,8 +1239,7 @@ PDC_Server_checkpoint() checkpoint_bulki = BULKI_init(5); // BULKI version number for validation - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("version_number", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING), BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); // checkpoint containers @@ -1256,21 +1255,17 @@ PDC_Server_checkpoint() BULKI *container_entry = BULKI_init(2); hash_key = PDC_get_hash_by_name(cont_head->cont_name); - BULKI_put(container_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // store the container structure as binary blob - BULKI_put(container_entry, - BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_put(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(containers_array, container_entry); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("containers", PDC_STRING), - containers_array); + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); @@ -1286,13 +1281,11 @@ PDC_Server_checkpoint() BULKI *hash_entry = BULKI_init(3); // Store number of objects - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("n_obj", PDC_STRING), + BULKI_put(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING), BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); hash_key = PDC_get_hash_by_name(head->metadata->obj_name); - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // array of metadata objects @@ -1304,8 +1297,7 @@ PDC_Server_checkpoint() BULKI *metadata_obj = BULKI_init(3); // store metadata structure - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("metadata", PDC_STRING), + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); // kv tags @@ -1316,29 +1308,23 @@ PDC_Server_checkpoint() { BULKI *kvtag_entry = BULKI_init(4); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("key", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("size", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("type", PDC_STRING), + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, - BULKI_singleton_ENTITY("value", PDC_STRING), - BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, - PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, PDC_UINT8, + PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(kvtags_array, kvtag_entry); } - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("kvtags", PDC_STRING), - kvtags_array); + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); // storage regions n_region = 0; @@ -1352,8 +1338,7 @@ PDC_Server_checkpoint() BULKI *region_entry = BULKI_init(3); // store region structure - BULKI_put(region_entry, - BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); // store histogram if present @@ -1361,37 +1346,30 @@ PDC_Server_checkpoint() if (region_elt->region_hist != NULL) has_hist = 1; - BULKI_put(region_entry, - BULKI_singleton_ENTITY("has_hist", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING), BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); if (has_hist == 1) { BULKI *histogram = BULKI_init(5); - BULKI_put(histogram, - BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("range", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("range", PDC_STRING), BULKI_ENTITY(region_elt->region_hist->range, region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("bin", PDC_STRING), - BULKI_ENTITY(region_elt->region_hist->bin, - region_elt->region_hist->nbin, PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_put(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING), + BULKI_ENTITY(region_elt->region_hist->bin, region_elt->region_hist->nbin, + PDC_UINT64, PDC_CLS_ARRAY)); - BULKI_put(histogram, - BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_put(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); - BULKI_put(region_entry, - BULKI_singleton_ENTITY("histogram", PDC_STRING), + BULKI_put(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING), BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } @@ -1403,9 +1381,7 @@ PDC_Server_checkpoint() LOG_ERROR("Error with number of regions\n"); } - BULKI_put(metadata_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING), - regions_array); + BULKI_put(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); BULKI_ENTITY_append_BULKI(metadata_objs_array, metadata_obj); @@ -1413,15 +1389,12 @@ PDC_Server_checkpoint() region_count += n_region; } // end for metadata entry linked list - BULKI_put(hash_entry, - BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), - metadata_objs_array); + BULKI_put(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); } // end for hash table metadata entry - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), metadata_entries_array); // data server regions @@ -1434,8 +1407,7 @@ PDC_Server_checkpoint() { BULKI *dataserver_obj = BULKI_init(2); - BULKI_put(dataserver_obj, - BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); DL_COUNT(region->region_storage_head, region_elt, n_region); @@ -1444,34 +1416,29 @@ PDC_Server_checkpoint() DL_FOREACH(region->region_storage_head, region_elt) { BULKI *ds_region = BULKI_init(1); - BULKI_put(ds_region, - BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(ds_regions_array, ds_region); } - BULKI_put(dataserver_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING), - ds_regions_array); + BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), ds_regions_array); BULKI_ENTITY_append_BULKI(dataserver_regions_array, dataserver_obj); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), dataserver_regions_array); // transfer request metadata query BULKI *transfer_query_bulki = NULL; - ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); + ret_value = transfer_request_metadata_query_checkpoint_bulki(&transfer_query_bulki); if (ret_value != SUCCEED || transfer_query_bulki == NULL) { LOG_ERROR("Failed to create transfer query checkpoint\n"); PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); } - BULKI_put(checkpoint_bulki, - BULKI_singleton_ENTITY("transfer_query", PDC_STRING), + BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== @@ -1486,7 +1453,7 @@ PDC_Server_checkpoint() // note: BULKI_Entity_serialize_to_file() still do in-memory serialization // note: need to implement a streaming file writing without in-memory serialization BULKI_serialize_to_file(checkpoint_bulki, file); - file = NULL; // file was closed by BULKI_serialize_to_file() + file = NULL; // file was closed by BULKI_serialize_to_file() // Clean up BULKI_free(checkpoint_bulki, 1); @@ -1517,8 +1484,8 @@ PDC_Server_checkpoint() #endif #ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", - metadata_size, region_count, checkpoint_time_rank); + LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, + checkpoint_time_rank); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); @@ -1561,9 +1528,9 @@ PDC_Server_restart(char *filename) { FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; - int all_cont; + perr_t ret_value = SUCCEED; + int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; + int all_cont; pdc_metadata_t * metadata, *elt; region_list_t * region_list; pdc_hash_table_entry_head * entry; @@ -1572,12 +1539,12 @@ PDC_Server_restart(char *filename) unsigned idx; // BULKI-specific variables - BULKI * checkpoint_bulki = NULL; - BULKI_Entity * containers_array = NULL; - BULKI_Entity * metadata_entries_array = NULL; - BULKI_Entity * dataserver_regions_array = NULL; - char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; - FILE * file = NULL; + BULKI * checkpoint_bulki = NULL; + BULKI_Entity *containers_array = NULL; + BULKI_Entity *metadata_entries_array = NULL; + BULKI_Entity *dataserver_regions_array = NULL; + char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; + FILE * file = NULL; #ifdef PDC_TIMING double start = MPI_Wtime(); @@ -1596,7 +1563,7 @@ PDC_Server_restart(char *filename) // note: BULKI_deserialize_from_file will close the file checkpoint_bulki = BULKI_deserialize_from_file(file); - file = NULL; // File was closed by BULKI_deserialize_from_file + file = NULL; // File was closed by BULKI_deserialize_from_file if (checkpoint_bulki == NULL) { LOG_ERROR("Failed to deserialize checkpoint file\n"); @@ -1604,36 +1571,36 @@ PDC_Server_restart(char *filename) } // read and validate bulki version - BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); - int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_Entity *version_entity = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); + int equal = BULKI_Entity_equal(version_entity, + BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); if (!equal) { - LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", - PDC_CHECKPOINT_MAGIC_CURRENT, + LOG_ERROR("Checkpoint version mismatch: expected '%s', found '%s'\n", PDC_CHECKPOINT_MAGIC_CURRENT, version_entity ? (char *)version_entity->data : "NULL"); PGOTO_ERROR(FAIL, "Checkpoint version mismatch"); } // restore containers - containers_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("containers", PDC_STRING)); + containers_array = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING)); if (containers_array != NULL && containers_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *cont_iter = Bent_iterator_init(containers_array, NULL, PDC_BULKI); - all_cont = containers_array->count; + all_cont = containers_array->count; while (Bent_iterator_has_next_BULKI(cont_iter)) { BULKI *container_entry = Bent_iterator_next_BULKI(cont_iter); // Extract hash key - BULKI_Entity *hash_key_ent = BULKI_get(container_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + BULKI_Entity *hash_key_ent = + BULKI_get(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); // Extract container data - BULKI_Entity *cont_data_ent = BULKI_get(container_entry, - BULKI_singleton_ENTITY("cont_data", PDC_STRING)); + BULKI_Entity *cont_data_ent = + BULKI_get(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING)); cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); memcpy(cont_entry, cont_data_ent->data, sizeof(pdc_cont_hash_table_entry_t)); total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); @@ -1652,8 +1619,8 @@ PDC_Server_restart(char *filename) } // restore metadata - metadata_entries_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); + metadata_entries_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING)); if (metadata_entries_array != NULL && metadata_entries_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *entry_iter = Bent_iterator_init(metadata_entries_array, NULL, PDC_BULKI); @@ -1662,14 +1629,13 @@ PDC_Server_restart(char *filename) BULKI *hash_entry = Bent_iterator_next_BULKI(entry_iter); // extract n_obj - BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("n_obj", PDC_STRING)); - int count; + BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING)); + int count; memcpy(&count, n_obj_ent->data, sizeof(int)); // extract hash key - BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + BULKI_Entity *hash_key_ent = + BULKI_get(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); @@ -1684,19 +1650,19 @@ PDC_Server_restart(char *filename) metadata = (pdc_metadata_t *)PDC_calloc(sizeof(pdc_metadata_t), count); // extract metadata objects array - BULKI_Entity *metadata_objs_array = BULKI_get(hash_entry, - BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); + BULKI_Entity *metadata_objs_array = + BULKI_get(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING)); if (metadata_objs_array != NULL && metadata_objs_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *obj_iter = Bent_iterator_init(metadata_objs_array, NULL, PDC_BULKI); - i = 0; + i = 0; while (Bent_iterator_has_next_BULKI(obj_iter) && i < count) { BULKI *metadata_obj = Bent_iterator_next_BULKI(obj_iter); // extract metadata structure - BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("metadata", PDC_STRING)); + BULKI_Entity *metadata_ent = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING)); memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); // initialize pointers @@ -1711,8 +1677,8 @@ PDC_Server_restart(char *filename) (metadata + i)->all_storage_region_distributed = 0; // restore kv tags - BULKI_Entity *kvtags_array = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("kvtags", PDC_STRING)); + BULKI_Entity *kvtags_array = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING)); if (kvtags_array != NULL && kvtags_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *kvtag_iter = Bent_iterator_init(kvtags_array, NULL, PDC_BULKI); @@ -1720,29 +1686,30 @@ PDC_Server_restart(char *filename) while (Bent_iterator_has_next_BULKI(kvtag_iter)) { BULKI *kvtag_entry = Bent_iterator_next_BULKI(kvtag_iter); - pdc_kvtag_list_t *kvtag_list = (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); - kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); + pdc_kvtag_list_t *kvtag_list = + (pdc_kvtag_list_t *)PDC_calloc(1, sizeof(pdc_kvtag_list_t)); + kvtag_list->kvtag = (pdc_kvtag_t *)PDC_malloc(sizeof(pdc_kvtag_t)); // extract key - BULKI_Entity *key_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("key", PDC_STRING)); - int key_len = strlen((char *)key_ent->data) + 1; + BULKI_Entity *key_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING)); + int key_len = strlen((char *)key_ent->data) + 1; kvtag_list->kvtag->name = PDC_malloc(key_len); memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); // extract size - BULKI_Entity *size_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("size", PDC_STRING)); + BULKI_Entity *size_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING)); memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); // extract type - BULKI_Entity *type_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("type", PDC_STRING)); + BULKI_Entity *type_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING)); memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); // extract value - BULKI_Entity *value_ent = BULKI_get(kvtag_entry, - BULKI_singleton_ENTITY("value", PDC_STRING)); + BULKI_Entity *value_ent = + BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING)); kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); memcpy(kvtag_list->kvtag->value, value_ent->data, kvtag_list->kvtag->size); @@ -1751,13 +1718,14 @@ PDC_Server_restart(char *filename) } // restore storage regions - BULKI_Entity *regions_array = BULKI_get(metadata_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *regions_array = + BULKI_get(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); int n_region = 0; if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { n_region = regions_array->count; - BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); + BULKI_Entity_Iterator *region_iter = + Bent_iterator_init(regions_array, NULL, PDC_BULKI); while (Bent_iterator_has_next_BULKI(region_iter)) { BULKI *region_entry = Bent_iterator_next_BULKI(region_iter); @@ -1765,53 +1733,54 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); // extract region structure - BULKI_Entity *region_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("region", PDC_STRING)); + BULKI_Entity *region_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // extract histogram flag - BULKI_Entity *has_hist_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("has_hist", PDC_STRING)); + BULKI_Entity *has_hist_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING)); int has_hist; memcpy(&has_hist, has_hist_ent->data, sizeof(int)); if (has_hist == 1) { - BULKI_Entity *histogram_ent = BULKI_get(region_entry, - BULKI_singleton_ENTITY("histogram", PDC_STRING)); + BULKI_Entity *histogram_ent = + BULKI_get(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING)); if (histogram_ent != NULL && histogram_ent->pdc_type == PDC_BULKI) { BULKI *histogram = (BULKI *)histogram_ent->data; - region_list->region_hist = (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); + region_list->region_hist = + (pdc_histogram_t *)PDC_malloc(sizeof(pdc_histogram_t)); - BULKI_Entity *dtype_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("dtype", PDC_STRING)); + BULKI_Entity *dtype_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING)); memcpy(®ion_list->region_hist->dtype, dtype_ent->data, sizeof(int)); - BULKI_Entity *nbin_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("nbin", PDC_STRING)); + BULKI_Entity *nbin_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING)); memcpy(®ion_list->region_hist->nbin, nbin_ent->data, sizeof(int)); if (region_list->region_hist->nbin == 0) { LOG_ERROR("Checkpoint file histogram size is 0\n"); } - BULKI_Entity *range_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("range", PDC_STRING)); - region_list->region_hist->range = - (double *)PDC_malloc(sizeof(double) * region_list->region_hist->nbin * 2); + BULKI_Entity *range_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("range", PDC_STRING)); + region_list->region_hist->range = (double *)PDC_malloc( + sizeof(double) * region_list->region_hist->nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, sizeof(double) * region_list->region_hist->nbin * 2); - BULKI_Entity *bin_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("bin", PDC_STRING)); - region_list->region_hist->bin = - (uint64_t *)PDC_malloc(sizeof(uint64_t) * region_list->region_hist->nbin); + BULKI_Entity *bin_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); + region_list->region_hist->bin = (uint64_t *)PDC_malloc( + sizeof(uint64_t) * region_list->region_hist->nbin); memcpy(region_list->region_hist->bin, bin_ent->data, sizeof(uint64_t) * region_list->region_hist->nbin); - BULKI_Entity *incr_ent = BULKI_get(histogram, - BULKI_singleton_ENTITY("incr", PDC_STRING)); + BULKI_Entity *incr_ent = + BULKI_get(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING)); memcpy(®ion_list->region_hist->incr, incr_ent->data, sizeof(double)); } } @@ -1845,7 +1814,8 @@ PDC_Server_restart(char *filename) region_list->io_cache_region = NULL; memset(region_list->shm_addr, 0, ADDR_MAX); - memset(region_list->client_ids, 0, PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); + memset(region_list->client_ids, 0, + PDC_SERVER_MAX_PROC_PER_NODE * sizeof(uint32_t)); if (strstr(region_list->storage_location, "/global/cscratch") != NULL) { region_list->data_loc_type = PDC_LUSTRE; @@ -1870,7 +1840,7 @@ PDC_Server_restart(char *filename) // insert metadata to hash table for (i = 0; i < count; i++) { - elt = metadata + i; + elt = metadata + i; ret_value = PDC_Server_hash_table_list_insert(entry, elt); if (ret_value != SUCCEED) PGOTO_ERROR(FAIL, "Error with hash table recovering from checkpoint file"); @@ -1879,8 +1849,8 @@ PDC_Server_restart(char *filename) } // restore data server regions - dataserver_regions_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); + dataserver_regions_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING)); if (dataserver_regions_array != NULL && dataserver_regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *ds_iter = Bent_iterator_init(dataserver_regions_array, NULL, PDC_BULKI); @@ -1894,13 +1864,13 @@ PDC_Server_restart(char *filename) new_obj_reg->storage_location = (char *)PDC_malloc(sizeof(char) * ADDR_MAX); // extract obj_id - BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, - BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + BULKI_Entity *obj_id_ent = + BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); // extract regions - BULKI_Entity *ds_regions_array = BULKI_get(dataserver_obj, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *ds_regions_array = + BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); if (ds_regions_array != NULL && ds_regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *ds_region_iter = Bent_iterator_init(ds_regions_array, NULL, PDC_BULKI); @@ -1910,8 +1880,8 @@ PDC_Server_restart(char *filename) region_list = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - BULKI_Entity *region_ent = BULKI_get(ds_region, - BULKI_singleton_ENTITY("region", PDC_STRING)); + BULKI_Entity *region_ent = + BULKI_get(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING)); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // initialize fields (similar to above) @@ -1939,8 +1909,8 @@ PDC_Server_restart(char *filename) } // restore transfer query - BULKI_Entity *transfer_query_ent = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); + BULKI_Entity *transfer_query_ent = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING)); if (transfer_query_ent != NULL && transfer_query_ent->pdc_type == PDC_BULKI) { // Extract the nested BULKI containing transfer query data @@ -1969,8 +1939,8 @@ PDC_Server_restart(char *filename) MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&total_region, &all_n_region, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else - all_nobj = nobj; - all_n_region = total_region; + all_nobj = nobj; + all_n_region = total_region; #endif if (pdc_server_rank_g == 0) { diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index 8a2c8dc8e..440aa763c 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -71,8 +71,8 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che pthread_mutex_init(&metadata_query_mutex, NULL); if (checkpoint_bulki != NULL) { - BULKI_Entity *objects_array = BULKI_get(checkpoint_bulki, - BULKI_singleton_ENTITY("objects", PDC_STRING)); + BULKI_Entity *objects_array = + BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("objects", PDC_STRING)); if (objects_array == NULL || objects_array->pdc_type != PDC_BULKI) { LOG_ERROR("Invalid transfer query checkpoint: missing or invalid 'objects' field\n"); @@ -84,11 +84,9 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che while (Bent_iterator_has_next_BULKI(obj_iter)) { BULKI *obj_bulki = Bent_iterator_next_BULKI(obj_iter); - pdc_obj_metadata_pkg *obj_pkg = - (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + pdc_obj_metadata_pkg *obj_pkg = (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); - BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + BULKI_Entity *obj_id_ent = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); if (obj_id_ent == NULL) { LOG_ERROR("Missing obj_id in checkpoint object\n"); PDC_free(obj_pkg); @@ -96,8 +94,7 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che } memcpy(&obj_pkg->obj_id, obj_id_ent->data, sizeof(uint64_t)); - BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("ndim", PDC_STRING)); + BULKI_Entity *ndim_ent = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING)); if (ndim_ent == NULL) { LOG_ERROR("Missing ndim in checkpoint object\n"); PDC_free(obj_pkg); @@ -105,12 +102,11 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che } memcpy(&obj_pkg->ndim, ndim_ent->data, sizeof(int)); - obj_pkg->regions = NULL; + obj_pkg->regions = NULL; obj_pkg->regions_end = NULL; - obj_pkg->next = NULL; + obj_pkg->next = NULL; - BULKI_Entity *regions_array = BULKI_get(obj_bulki, - BULKI_singleton_ENTITY("regions", PDC_STRING)); + BULKI_Entity *regions_array = BULKI_get(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING)); if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); @@ -122,25 +118,27 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); region_pkg->reg_offset = (uint64_t *)PDC_malloc(sizeof(uint64_t) * obj_pkg->ndim * 2); - region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; + region_pkg->reg_size = region_pkg->reg_offset + obj_pkg->ndim; - BULKI_Entity *server_id_ent = BULKI_get(region_bulki, - BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); + BULKI_Entity *server_id_ent = + BULKI_get(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING)); if (server_id_ent != NULL) { memcpy(®ion_pkg->data_server_id, server_id_ent->data, sizeof(uint32_t)); - } else { + } + else { LOG_ERROR("Missing data_server_id in checkpoint region\n"); PDC_free(region_pkg->reg_offset); PDC_free(region_pkg); continue; } - BULKI_Entity *offset_size_ent = BULKI_get(region_bulki, - BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); + BULKI_Entity *offset_size_ent = + BULKI_get(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING)); if (offset_size_ent != NULL) { memcpy(region_pkg->reg_offset, offset_size_ent->data, sizeof(uint64_t) * obj_pkg->ndim * 2); - } else { + } + else { LOG_ERROR("Missing reg_offset_size in checkpoint region\n"); PDC_free(region_pkg->reg_offset); PDC_free(region_pkg); @@ -150,21 +148,23 @@ transfer_request_metadata_query_init_bulki(int pdc_server_size_input, BULKI *che region_pkg->next = NULL; if (obj_pkg->regions == NULL) { - obj_pkg->regions = region_pkg; + obj_pkg->regions = region_pkg; obj_pkg->regions_end = region_pkg; - } else { + } + else { obj_pkg->regions_end->next = region_pkg; - obj_pkg->regions_end = region_pkg; + obj_pkg->regions_end = region_pkg; } } } if (metadata_server_objs == NULL) { - metadata_server_objs = obj_pkg; + metadata_server_objs = obj_pkg; metadata_server_objs_end = obj_pkg; - } else { + } + else { metadata_server_objs_end->next = obj_pkg; - metadata_server_objs_end = obj_pkg; + metadata_server_objs_end = obj_pkg; } } @@ -298,28 +298,26 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) pdc_obj_metadata_pkg * obj_temp; pdc_region_metadata_pkg *region_temp; int obj_count = 0; - BULKI * bulki = NULL; + BULKI * bulki = NULL; -// if (checkpoint_bulki == NULL) { -// LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); -// PGOTO_ERROR(FAIL, "Invalid parameter"); -// } + // if (checkpoint_bulki == NULL) { + // LOG_ERROR("checkpoint_bulki output parameter is NULL\n"); + // PGOTO_ERROR(FAIL, "Invalid parameter"); + // } pthread_mutex_lock(&metadata_query_mutex); - bulki = BULKI_init(1); + bulki = BULKI_init(1); BULKI_Entity *objects_array = empty_BULKI_Array_Entity(); obj_temp = metadata_server_objs; while (obj_temp) { BULKI *obj_bulki = BULKI_init(3); - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("ndim", PDC_STRING), + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING), BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); @@ -328,22 +326,17 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) while (region_temp) { BULKI *region_bulki = BULKI_init(2); - BULKI_put(region_bulki, - BULKI_singleton_ENTITY("data_server_id", PDC_STRING), + BULKI_put(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING), BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(region_bulki, - BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), - BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, - PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_put(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, PDC_UINT64, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI(regions_array, region_bulki); region_temp = region_temp->next; } - BULKI_put(obj_bulki, - BULKI_singleton_ENTITY("regions", PDC_STRING), - regions_array); + BULKI_put(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); BULKI_ENTITY_append_BULKI(objects_array, obj_bulki); @@ -351,9 +344,7 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) obj_temp = obj_temp->next; } - BULKI_put(bulki, - BULKI_singleton_ENTITY("objects", PDC_STRING), - objects_array); + BULKI_put(bulki, BULKI_singleton_ENTITY("objects", PDC_STRING), objects_array); pthread_mutex_unlock(&metadata_query_mutex); diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index a7ffd21c1..ab98d093d 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -68,23 +68,23 @@ print_usage(char *name) int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont; - pdcid_t * obj_ids; - int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; - int obj_1percent = 0; - int proc_num, my_rank, i; - char obj_name[128]; - double stime, total_time, percent_time; - pdc_kvtag_t kvtag; - void ** values; + pdcid_t pdc, cont_prop, cont; + pdcid_t * obj_ids; + int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; + int obj_1percent = 0; + int proc_num, my_rank, i; + char obj_name[128]; + double stime, total_time, percent_time; + pdc_kvtag_t kvtag; + void ** values; pdc_var_type_t value_type; size_t value_size; int ret_value = SUCCEED; // counters for verification statistics - int verified_success = 0; - int verified_fail = 0; - int all_verified_success = 0; - int all_verified_fail = 0; + int verified_success = 0; + int verified_fail = 0; + int all_verified_success = 0; + int all_verified_fail = 0; #ifdef ENABLE_MPI MPI_Init(&argc, &argv); @@ -175,7 +175,7 @@ main(int argc, char *argv[]) (void *)&value_size) < 0) PGOTO_ERROR(FAIL, "Failed to get a kvtag from o%d\n", i + n_tag_s); - int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 + int expected_value = i + n_tag_s; // Assuming tags were added in order starting from 0 // count successful and failed verifications instead of immediate error if (*(int *)(values[i]) == expected_value) { @@ -185,8 +185,8 @@ main(int argc, char *argv[]) verified_fail++; // log first 10 failures for debugging if (verified_fail <= 10) { - LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", - i + n_tag_s, expected_value, *(int *)(values[i])); + LOG_ERROR("Verification failed for obj%d: expected %d, got %d\n", i + n_tag_s, expected_value, + *(int *)(values[i])); } } free(values[i]); @@ -197,8 +197,8 @@ main(int argc, char *argv[]) total_time = MPI_Wtime() - stime; #endif if (my_rank == 0) - LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, n_obj, - total_time, n_query / total_time); + LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, + n_obj, total_time, n_query / total_time); free(values); free(obj_ids); @@ -209,7 +209,7 @@ main(int argc, char *argv[]) MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else all_verified_success = verified_success; - all_verified_fail = verified_fail; + all_verified_fail = verified_fail; #endif if (my_rank == 0) { @@ -225,7 +225,8 @@ main(int argc, char *argv[]) if (all_verified_fail > 0) { LOG_ERROR("WARNING: %d objects failed verification!\n", all_verified_fail); - } else { + } + else { LOG_INFO("SUCCESS: All objects verified correctly!\n"); } } From ca6b6369ee9f7931ff4b918775d04dc1c4d49872 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:02:00 -0400 Subject: [PATCH 10/30] Move the bulki logs under build preprocessor --- src/commons/serde/bulki/bulki.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index b9d4deaeb..372a784b5 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -624,7 +624,9 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) for (size_t i = 0; i < bulk_entity->count; i++) { BULKI_free(&bulki_array[i], 0); } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_array 1\n"); +#endif bulki_array = NULL; } else if (bulk_entity->pdc_type == PDC_BULKI_ENT && bulk_entity->data != NULL) { @@ -632,7 +634,9 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) for (size_t i = 0; i < bulk_entity->count; i++) { BULKI_Entity_free(&bulki_entity_array[i], 0); } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_array 2\n"); +#endif bulki_entity_array = NULL; } } @@ -640,15 +644,21 @@ BULKI_Entity_free(BULKI_Entity *bulk_entity, int free_struct) if (bulk_entity->pdc_type == PDC_BULKI && bulk_entity->data != NULL) { BULKI_free((BULKI *)bulk_entity->data, 0); bulk_entity->data = NULL; +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulki_item 1\n"); +#endif } } +#ifdef BUKLI_DEBUG_LOG LOG_INFO("Freeing bulk_entity\n"); +#endif if (bulk_entity->data != NULL) { +#ifdef BUKLI_DEBUG_LOG LOG_INFO( "bulki_entity->class: %d, bulki_entity->class: %d, bulki_entity->data: %p, bulki_entity: " "%p\n", bulk_entity->pdc_class, bulk_entity->pdc_type, bulk_entity->data, bulk_entity); +#endif bulk_entity->data = (void *)PDC_free(bulk_entity->data); bulk_entity->data = NULL; } From b995ac85bdc4535c27b55d910c61d7379aa3fcbe Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:02:46 -0400 Subject: [PATCH 11/30] Add timer log for server restart --- src/server/pdc_server.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 1b0d1ec21..5a21a3e02 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1579,7 +1579,7 @@ PDC_Server_restart(char *filename) char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; FILE * file = NULL; -#ifdef PDC_TIMING +#if defined(PDC_TIMING) || defined(ENABLE_MPI) double start = MPI_Wtime(); #endif @@ -1983,6 +1983,13 @@ PDC_Server_restart(char *filename) #ifdef PDC_TIMING pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; #endif + +#ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); + if (!pdc_server_rank_g) { + LOG_INFO("total restart time = %lf\n", MPI_Wtime() - start); + } +#endif // ensure file is closed if error occurred before BULKI_deserialize_from_file if (file != NULL) { fclose(file); From 0b76afce885b1ae3a8adff37f9f121db05b380ae Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 26 Mar 2026 17:04:01 -0400 Subject: [PATCH 12/30] Fix issue in printing server close time --- src/api/close_server.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/close_server.c b/src/api/close_server.c index 853826980..96f34c58a 100644 --- a/src/api/close_server.c +++ b/src/api/close_server.c @@ -42,11 +42,12 @@ main(int argc, char *argv[]) int rank; double start; MPI_Init(&argc, &argv); - +#endif + pdc = PDCinit("pdc"); +#ifdef ENABLE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &rank); start = MPI_Wtime(); #endif - pdc = PDCinit("pdc"); PDC_Client_close_all_server(); @@ -54,6 +55,7 @@ main(int argc, char *argv[]) LOG_ERROR("Failed to close PDC\n"); #ifdef ENABLE_MPI + MPI_Barrier(MPI_COMM_WORLD); if (!rank) { LOG_INFO("total close time = %lf\n", MPI_Wtime() - start); } From 08bff24f6c94d4661af6981a567c12ba9d331e9d Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 15 May 2026 01:34:57 -0400 Subject: [PATCH 13/30] Fix log level to report the total checkpoint time --- src/server/pdc_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 54892686d..2ec1f06ec 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1491,7 +1491,7 @@ PDC_Server_checkpoint() checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (pdc_server_rank_g == 0) - LOG_ERROR("Rank[ ALL]: Total checkpoint time = %.6f\n", checkpoint_time); + LOG_INFO("Rank[ ALL]: Total checkpoint time = %.6f\n", checkpoint_time); #endif if (pdc_server_rank_g == 0) { From 1b889a365bdb012dc1d872299fc0e7772005bbb2 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 13:42:47 -0400 Subject: [PATCH 14/30] Fix close_server MPI init order and remove redundant barrier Initialize MPI and start the close timer before PDCinit so client setup uses the correct communicator. Drop the extra MPI_Barrier before printing total close time since PDC_Client_close_all_server already synchronizes ranks after the close RPC completes. --- src/api/close_server.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/api/close_server.c b/src/api/close_server.c index 96f34c58a..535d847fa 100644 --- a/src/api/close_server.c +++ b/src/api/close_server.c @@ -42,12 +42,11 @@ main(int argc, char *argv[]) int rank; double start; MPI_Init(&argc, &argv); -#endif - pdc = PDCinit("pdc"); -#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); start = MPI_Wtime(); #endif + pdc = PDCinit("pdc"); PDC_Client_close_all_server(); @@ -55,7 +54,6 @@ main(int argc, char *argv[]) LOG_ERROR("Failed to close PDC\n"); #ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); if (!rank) { LOG_INFO("total close time = %lf\n", MPI_Wtime() - start); } @@ -63,4 +61,4 @@ main(int argc, char *argv[]) #endif FUNC_LEAVE(0); -} +} \ No newline at end of file From 8cea3f561793f49cbb26fc30432b7110dd9d95c4 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 13:51:22 -0400 Subject: [PATCH 15/30] Speed up BULKI checkpoint tree construction with incremental sizing APIs Add _incremental and _with_capacity BULKI helpers that update cached sizes in O(1) and pre-allocate array slots, while keeping the original put/append/delete APIs unchanged for existing callers. Switch PDC_Server_checkpoint and transfer-query checkpoint building to the fast path, document the new APIs, and add unit tests for incremental put, append, and delete sizing. Mark remaining serialize/write redundancies in bulki_serde.c for follow-up work. --- src/commons/serde/bulki/bulki.c | 254 ++++++++++++++++-- src/commons/serde/bulki/bulki_serde.c | 5 + src/commons/serde/bulki/bulki_serde_test.c | 225 ++++++++++++++++ src/commons/serde/bulki/include/bulki.h | 96 ++++++- src/server/pdc_server.c | 87 +++--- ...dc_server_region_transfer_metadata_query.c | 34 ++- 6 files changed, 630 insertions(+), 71 deletions(-) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index 372a784b5..77c20d007 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -4,6 +4,17 @@ #include "pdc_timing.h" #include "pdc_malloc.h" +/** Wire size of a BULKI_Entity header: pdc_class, pdc_type, count, size fields. */ +#define BULKI_ENTITY_WIRE_HEADER_SIZE ((size_t)(sizeof(int8_t) * 2 + sizeof(uint64_t) * 2)) +/** Wire size of a BULKI map header: totalSize, numKeys, headerSize, dataSize, offsets. */ +#define BULKI_WIRE_META_SIZE ((size_t)(sizeof(uint64_t) * 6)) + +static size_t bulki_entity_wire_size(BULKI_Entity *entity); +static size_t bulki_wire_size(BULKI *bulki); +static void bulki_refresh_total_size(BULKI *bulki); +static void bulki_entity_array_ensure_capacity(BULKI_Entity *dest, size_t element_size); +static BULKI_Entity *empty_array_entity(pdc_c_var_type_t pdc_type, int initial_capacity); + size_t get_BULKI_Entity_size(BULKI_Entity *bulk_entity) { @@ -69,6 +80,102 @@ get_BULKI_size(BULKI *bulki) FUNC_LEAVE(size); } +/** + * Return the serialized byte length of a BULKI_Entity for incremental size updates. + * + * Uses the cached `entity->size` when set; otherwise falls back to a full + * `get_BULKI_Entity_size()` walk. Used by `_incremental` append helpers to avoid + * recomputing sibling subtrees on each append. + */ +static size_t +bulki_entity_wire_size(BULKI_Entity *entity) +{ + if (entity != NULL && entity->size > 0) + return entity->size; + return get_BULKI_Entity_size(entity); +} + +/** + * Return the serialized byte length of a BULKI map for incremental size updates. + * + * Uses the cached `bulki->totalSize` when set; otherwise falls back to + * `get_BULKI_size()`. Used when appending a BULKI child to an array entity. + */ +static size_t +bulki_wire_size(BULKI *bulki) +{ + if (bulki != NULL && bulki->totalSize > 0) + return bulki->totalSize; + return get_BULKI_size(bulki); +} + +/** + * Recompute `bulki->totalSize` from cached header and data region sizes in O(1). + * + * Assumes `headerSize` and `dataSize` are already accurate (maintained by + * `BULKI_put_incremental` / `BULKI_delete_incremental`). Does not walk keys. + */ +static void +bulki_refresh_total_size(BULKI *bulki) +{ + bulki->totalSize = BULKI_WIRE_META_SIZE + bulki->header->headerSize + bulki->data->dataSize; +} + +/** + * Grow a BULKI_Entity array backing store when `count` reaches `capacity`. + * + * Doubles capacity (minimum 1) and reallocates `dest->data`. No-op when free + * slots remain. Used by `_incremental` append paths to avoid per-element realloc. + * + * @param dest Array entity (PDC_CLS_ARRAY, PDC_BULKI or PDC_BULKI_ENT) + * @param element_size sizeof(BULKI) or sizeof(BULKI_Entity) + */ +static void +bulki_entity_array_ensure_capacity(BULKI_Entity *dest, size_t element_size) +{ + if (dest->count < dest->capacity) + return; + + if (dest->capacity == 0) + dest->capacity = 1; + else + dest->capacity *= 2; + + dest->data = PDC_realloc(dest->data, dest->capacity * element_size); +} + +/** + * Allocate an empty array BULKI_Entity, optionally with pre-sized backing storage. + * + * Shared by `empty_BULKI_Array_Entity_with_capacity` and + * `empty_Bent_Array_Entity_with_capacity`. Initializes `size` to the wire header + * only; `_incremental` appends add child wire sizes as elements are inserted. + * + * @param pdc_type PDC_BULKI or PDC_BULKI_ENT + * @param initial_capacity Slot count to pre-allocate; 0 leaves `data` NULL + */ +static BULKI_Entity * +empty_array_entity(pdc_c_var_type_t pdc_type, int initial_capacity) +{ + BULKI_Entity *bulki_entity = (BULKI_Entity *)PDC_calloc(1, sizeof(BULKI_Entity)); + bulki_entity->pdc_type = pdc_type; + bulki_entity->pdc_class = PDC_CLS_ARRAY; + bulki_entity->count = 0; + bulki_entity->capacity = 0; + bulki_entity->data = NULL; + bulki_entity->size = BULKI_ENTITY_WIRE_HEADER_SIZE; + + if (initial_capacity > 0) { + bulki_entity->capacity = (uint64_t)initial_capacity; + if (pdc_type == PDC_BULKI) + bulki_entity->data = PDC_calloc((size_t)initial_capacity, sizeof(BULKI)); + else if (pdc_type == PDC_BULKI_ENT) + bulki_entity->data = PDC_calloc((size_t)initial_capacity, sizeof(BULKI_Entity)); + } + + return bulki_entity; +} + void BULKI_Entity_print(BULKI_Entity *bulk_entity) { @@ -147,30 +254,28 @@ BULKI_Entity * empty_BULKI_Array_Entity() { FUNC_ENTER(NULL); + FUNC_LEAVE(empty_array_entity(PDC_BULKI, 0)); +} - BULKI_Entity *bulki_entity = (BULKI_Entity *)PDC_calloc(1, sizeof(BULKI_Entity)); - bulki_entity->pdc_type = PDC_BULKI; - bulki_entity->pdc_class = PDC_CLS_ARRAY; - bulki_entity->count = 0; - bulki_entity->data = NULL; - get_BULKI_Entity_size(bulki_entity); - - FUNC_LEAVE(bulki_entity); +BULKI_Entity * +empty_BULKI_Array_Entity_with_capacity(int initial_capacity) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(empty_array_entity(PDC_BULKI, initial_capacity)); } BULKI_Entity * empty_Bent_Array_Entity() { FUNC_ENTER(NULL); + FUNC_LEAVE(empty_array_entity(PDC_BULKI_ENT, 0)); +} - BULKI_Entity *bulki_entity = (BULKI_Entity *)PDC_calloc(1, sizeof(BULKI_Entity)); - bulki_entity->pdc_type = PDC_BULKI_ENT; - bulki_entity->pdc_class = PDC_CLS_ARRAY; - bulki_entity->count = 0; - bulki_entity->data = NULL; - get_BULKI_Entity_size(bulki_entity); - - FUNC_LEAVE(bulki_entity); +BULKI_Entity * +empty_Bent_Array_Entity_with_capacity(int initial_capacity) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(empty_array_entity(PDC_BULKI_ENT, initial_capacity)); } BULKI_Entity * @@ -194,6 +299,31 @@ BULKI_ENTITY_append_BULKI(BULKI_Entity *dest, BULKI *src) FUNC_LEAVE(dest); } +BULKI_Entity * +BULKI_ENTITY_append_BULKI_incremental(BULKI_Entity *dest, BULKI *src) +{ + FUNC_ENTER(NULL); + + size_t child_size; + + if (src == NULL || dest == NULL) { + LOG_ERROR("Error: bulki is NULL\n"); + FUNC_LEAVE(NULL); + } + if (dest->pdc_class != PDC_CLS_ARRAY || dest->pdc_type != PDC_BULKI) { + LOG_ERROR("Error: dest is not an array of BULKI structure\n"); + FUNC_LEAVE(NULL); + } + + child_size = bulki_wire_size(src); + bulki_entity_array_ensure_capacity(dest, sizeof(BULKI)); + memcpy((BULKI *)dest->data + dest->count, src, sizeof(BULKI)); + dest->size += child_size; + dest->count++; + + FUNC_LEAVE(dest); +} + BULKI * BULKI_ENTITY_get_BULKI(BULKI_Entity *bulki_entity, size_t idx) { @@ -237,6 +367,31 @@ BULKI_ENTITY_append_BULKI_Entity(BULKI_Entity *dest, BULKI_Entity *src) FUNC_LEAVE(dest); } +BULKI_Entity * +BULKI_ENTITY_append_BULKI_Entity_incremental(BULKI_Entity *dest, BULKI_Entity *src) +{ + FUNC_ENTER(NULL); + + size_t child_size; + + if (src == NULL || dest == NULL) { + LOG_ERROR("Error: bulki is NULL\n"); + FUNC_LEAVE(NULL); + } + if (dest->pdc_class != PDC_CLS_ARRAY || dest->pdc_type != PDC_BULKI_ENT) { + LOG_ERROR("Error: dest is not an array of BULKI_Entity structure\n"); + FUNC_LEAVE(NULL); + } + + child_size = bulki_entity_wire_size(src); + bulki_entity_array_ensure_capacity(dest, sizeof(BULKI_Entity)); + memcpy((BULKI_Entity *)dest->data + dest->count, src, sizeof(BULKI_Entity)); + dest->size += child_size; + dest->count++; + + FUNC_LEAVE(dest); +} + BULKI_Entity * BULKI_ENTITY_get_BULKI_Entity(BULKI_Entity *bulki_entity, size_t idx) { @@ -280,6 +435,10 @@ BULKI_ENTITY(void *data, uint64_t count, pdc_c_var_type_t pdc_type, pdc_c_var_cl bulki_entity->data = data; } else { + // note: BULKI_ENTITY currently deep-copies payload into BULKI-owned memory. + // note: this copy can be avoided by introducing explicit pointer ownership semantics + // note: (e.g., borrowed/non-owned data mode) so BULKI can reference external buffers + // note: without freeing them in BULKI_Entity_free(). bulki_entity->data = PDC_calloc(1, size); memcpy(bulki_entity->data, data, size); } @@ -317,7 +476,7 @@ BULKI_init(int initial_field_count) buiki->data = PDC_calloc(1, sizeof(BULKI_Data)); buiki->data->values = PDC_calloc(buiki->capacity, sizeof(BULKI_Entity)); buiki->data->dataSize = 0; - get_BULKI_size(buiki); + bulki_refresh_total_size(buiki); FUNC_LEAVE(buiki); } @@ -434,6 +593,45 @@ BULKI_put(BULKI *bulki, BULKI_Entity *key, BULKI_Entity *value) FUNC_LEAVE_VOID(); } +void +BULKI_put_incremental(BULKI *bulki, BULKI_Entity *key, BULKI_Entity *value) +{ + FUNC_ENTER(NULL); + + if (bulki == NULL || key == NULL || value == NULL) { + LOG_ERROR("Error: bulki, key, or value is NULL\n"); + FUNC_LEAVE_VOID(); + } + // search for existing key + BULKI_Entity *existing_value = BULKI_get(bulki, key); + if (existing_value != NULL) { + bulki->header->headerSize -= key->size; + bulki->data->dataSize -= existing_value->size; + memcpy(existing_value, value, sizeof(BULKI_Entity)); + bulki->header->headerSize += key->size; + bulki->data->dataSize += value->size; + bulki_refresh_total_size(bulki); + FUNC_LEAVE_VOID(); + } + if (bulki->numKeys >= bulki->capacity) { + bulki->capacity *= 2; + bulki->header->keys = PDC_realloc(bulki->header->keys, bulki->capacity * sizeof(BULKI_Entity)); + bulki->data->values = PDC_realloc(bulki->data->values, bulki->capacity * sizeof(BULKI_Entity)); + } + memcpy(&bulki->header->keys[bulki->numKeys], key, sizeof(BULKI_Entity)); + // append bytes for type, size, and key + bulki->header->headerSize += key->size; + + memcpy(&bulki->data->values[bulki->numKeys], value, sizeof(BULKI_Entity)); + // append bytes for class, type, size, and data + bulki->data->dataSize += value->size; + + bulki->numKeys++; + bulki_refresh_total_size(bulki); + + FUNC_LEAVE_VOID(); +} + BULKI_Entity * BULKI_delete(BULKI *bulki, BULKI_Entity *key) { @@ -456,6 +654,28 @@ BULKI_delete(BULKI *bulki, BULKI_Entity *key) FUNC_LEAVE(value); } +BULKI_Entity * +BULKI_delete_incremental(BULKI *bulki, BULKI_Entity *key) +{ + FUNC_ENTER(NULL); + + BULKI_Entity *value = NULL; + for (size_t i = 0; i < bulki->numKeys; i++) { + if (BULKI_Entity_equal(&bulki->header->keys[i], key)) { + value = &bulki->data->values[i]; + bulki->header->headerSize -= key->size; + bulki->data->dataSize -= value->size; + bulki->numKeys--; + memcpy(&bulki->header->keys[i], &bulki->header->keys[bulki->numKeys - 1], sizeof(BULKI_Entity)); + memcpy(&bulki->data->values[i], &bulki->data->values[bulki->numKeys - 1], sizeof(BULKI_Entity)); + break; + } + } + bulki_refresh_total_size(bulki); + + FUNC_LEAVE(value); +} + BULKI_Entity_Iterator * Bent_iterator_init(BULKI_Entity *array, void *filter, pdc_c_var_type_t filter_type) { diff --git a/src/commons/serde/bulki/bulki_serde.c b/src/commons/serde/bulki/bulki_serde.c index c55851cc1..2b03e00db 100644 --- a/src/commons/serde/bulki/bulki_serde.c +++ b/src/commons/serde/bulki/bulki_serde.c @@ -68,6 +68,7 @@ BULKI_Entity_serialize_to_buffer(BULKI_Entity *entity, void *buffer, size_t *off FUNC_ENTER(NULL); // serialize the size + // note: redundant tree walk — entity->size is already cached after incremental build; could use it directly uint64_t size = (uint64_t)get_BULKI_Entity_size(entity); *offset += BULKI_vle_encode_uint(size, buffer + *offset); @@ -122,6 +123,7 @@ BULKI_Entity_serialize(BULKI_Entity *entity, size_t *size) { FUNC_ENTER(NULL); + // note: redundant tree walk — entity->size is already cached; serialize_to_buffer recomputes per entity below size_t estimated_size = get_BULKI_Entity_size(entity); void * buffer = PDC_calloc(1, estimated_size); size_t offset = 0; @@ -180,6 +182,7 @@ BULKI_serialize(BULKI *data, size_t *size) { FUNC_ENTER(NULL); + // note: redundant tree walk — data->totalSize is already cached after BULKI_put_incremental size_t estimated_size = get_BULKI_size(data); void * buffer = PDC_calloc(1, estimated_size); size_t offset = 0; @@ -201,6 +204,7 @@ BULKI_Entity_serialize_to_file(BULKI_Entity *entity, FILE *fp) FUNC_ENTER(NULL); size_t size; + // note: buffer-then-fwrite — builds entire entity in memory before a single fwrite; consider streaming to fp void * buffer = BULKI_Entity_serialize(entity, &size); fwrite(buffer, size, 1, fp); buffer = (void *)PDC_free(buffer); @@ -215,6 +219,7 @@ BULKI_serialize_to_file(BULKI *bulki, FILE *fp) FUNC_ENTER(NULL); size_t size = 0; + // note: buffer-then-fwrite — full checkpoint serialized to RAM first; consider streaming serialize_to_file void * buffer = BULKI_serialize(bulki, &size); fwrite(buffer, size, 1, fp); buffer = (void *)PDC_free(buffer); diff --git a/src/commons/serde/bulki/bulki_serde_test.c b/src/commons/serde/bulki/bulki_serde_test.c index 0f2a7b79f..59827c278 100644 --- a/src/commons/serde/bulki/bulki_serde_test.c +++ b/src/commons/serde/bulki/bulki_serde_test.c @@ -274,6 +274,225 @@ test_bulki_in_entitiy() FUNC_LEAVE(equal); } +int +test_incremental_array_append_sizes() +{ + FUNC_ENTER(NULL); + + const int n_items = 128; + BULKI_Entity *arr = empty_BULKI_Array_Entity_with_capacity(n_items); + int i; + int pass = 1; + + for (i = 0; i < n_items; i++) { + BULKI *item = BULKI_init(1); + int val = i; + + BULKI_put_incremental(item, BULKI_ENTITY("key", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY(&val, 1, PDC_INT, PDC_CLS_ITEM)); + + { + size_t size_before = arr->size; + size_t expected_step = item->totalSize; + + BULKI_ENTITY_append_BULKI_incremental(arr, item); + if (arr->size != size_before + expected_step) { + LOG_ERROR("incremental array size mismatch at i=%d\n", i); + pass = 0; + break; + } + } + } + + if (pass) { + size_t recomputed = get_BULKI_Entity_size(arr); + if (arr->size != recomputed || arr->count != (uint64_t)n_items) { + LOG_ERROR("array size/count mismatch after append\n"); + pass = 0; + } + } + + if (pass && arr->capacity < (uint64_t)n_items) { + LOG_ERROR("preallocated capacity too small: %zu\n", (size_t)arr->capacity); + pass = 0; + } + + if (arr != NULL) + BULKI_Entity_free(arr, 1); + + FUNC_LEAVE(pass); +} + +int +test_preallocated_array_roundtrip() +{ + FUNC_ENTER(NULL); + + const int n_items = 64; + BULKI * root = BULKI_init(1); + BULKI_Entity *arr = empty_BULKI_Array_Entity_with_capacity(n_items); + int i; + + for (i = 0; i < n_items; i++) { + BULKI *item = BULKI_init(2); + int val = i * 10; + + BULKI_put_incremental(item, BULKI_singleton_ENTITY("name", PDC_STRING), + BULKI_singleton_ENTITY("entry", PDC_STRING)); + BULKI_put_incremental(item, BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_ENTITY(&val, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_ENTITY_append_BULKI_incremental(arr, item); + } + + BULKI_put_incremental(root, BULKI_singleton_ENTITY("entries", PDC_STRING), arr); + + { + size_t size; + void * buffer = BULKI_serialize(root, &size); + BULKI * deserialized = BULKI_deserialize(buffer); + int equal = BULKI_equal(root, deserialized); + + LOG_INFO("preallocated array roundtrip equal: %d\n", equal); + BULKI_free(deserialized, 1); + buffer = (void *)PDC_free(buffer); + BULKI_free(root, 1); + + FUNC_LEAVE(equal); + } +} + +int +test_incremental_bulki_put_total_size() +{ + FUNC_ENTER(NULL); + + BULKI *bulki = BULKI_init(4); + int pass = 1; + + BULKI_put_incremental(bulki, BULKI_ENTITY("a", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("1", 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_put_incremental(bulki, BULKI_ENTITY("b", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("2", 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_put_incremental(bulki, BULKI_ENTITY("c", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("3", 1, PDC_STRING, PDC_CLS_ITEM)); + + if (bulki->totalSize != get_BULKI_size(bulki)) { + LOG_ERROR("incremental totalSize mismatch after BULKI_put_incremental\n"); + pass = 0; + } + + BULKI_put_incremental(bulki, BULKI_ENTITY("a", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("updated", 1, PDC_STRING, PDC_CLS_ITEM)); + if (bulki->totalSize != get_BULKI_size(bulki)) { + LOG_ERROR("incremental totalSize mismatch after BULKI_put_incremental replace\n"); + pass = 0; + } + + BULKI_free(bulki, 1); + FUNC_LEAVE(pass); +} + +int +test_incremental_entity_array_append_sizes() +{ + FUNC_ENTER(NULL); + + const int n_items = 64; + BULKI_Entity *arr = empty_Bent_Array_Entity_with_capacity(n_items); + int i; + int pass = 1; + + for (i = 0; i < n_items; i++) { + int val = i * 3; + BULKI_Entity *item = BULKI_ENTITY(&val, 1, PDC_INT, PDC_CLS_ITEM); + + { + size_t size_before = arr->size; + size_t expected_step = item->size; + + BULKI_ENTITY_append_BULKI_Entity_incremental(arr, item); + if (arr->size != size_before + expected_step) { + LOG_ERROR("incremental entity array size mismatch at i=%d\n", i); + pass = 0; + break; + } + } + } + + if (pass) { + size_t recomputed = get_BULKI_Entity_size(arr); + if (arr->size != recomputed || arr->count != (uint64_t)n_items) { + LOG_ERROR("entity array size/count mismatch after append\n"); + pass = 0; + } + } + + if (pass && arr->capacity < (uint64_t)n_items) { + LOG_ERROR("preallocated entity array capacity too small: %zu\n", (size_t)arr->capacity); + pass = 0; + } + + if (arr != NULL) + BULKI_Entity_free(arr, 1); + + FUNC_LEAVE(pass); +} + +int +test_incremental_bulki_delete_total_size() +{ + FUNC_ENTER(NULL); + + BULKI *bulki = BULKI_init(4); + int pass = 1; + + BULKI_put_incremental(bulki, BULKI_ENTITY("a", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("1", 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_put_incremental(bulki, BULKI_ENTITY("b", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("2", 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_put_incremental(bulki, BULKI_ENTITY("c", 1, PDC_STRING, PDC_CLS_ITEM), + BULKI_ENTITY("3", 1, PDC_STRING, PDC_CLS_ITEM)); + + if (bulki->numKeys != 3 || bulki->totalSize != get_BULKI_size(bulki)) { + LOG_ERROR("incremental setup failed before BULKI_delete_incremental\n"); + pass = 0; + goto done; + } + + if (BULKI_delete_incremental(bulki, BULKI_ENTITY("b", 1, PDC_STRING, PDC_CLS_ITEM)) == NULL) { + LOG_ERROR("BULKI_delete_incremental did not remove existing key\n"); + pass = 0; + goto done; + } + + if (bulki->numKeys != 2 || bulki->totalSize != get_BULKI_size(bulki)) { + LOG_ERROR("incremental totalSize mismatch after BULKI_delete_incremental\n"); + pass = 0; + goto done; + } + + if (BULKI_get(bulki, BULKI_ENTITY("b", 1, PDC_STRING, PDC_CLS_ITEM)) != NULL) { + LOG_ERROR("deleted key still present after BULKI_delete_incremental\n"); + pass = 0; + goto done; + } + + if (BULKI_delete_incremental(bulki, BULKI_ENTITY("missing", 1, PDC_STRING, PDC_CLS_ITEM)) != NULL) { + LOG_ERROR("BULKI_delete_incremental should return NULL for missing key\n"); + pass = 0; + goto done; + } + + if (bulki->numKeys != 2 || bulki->totalSize != get_BULKI_size(bulki)) { + LOG_ERROR("incremental totalSize mismatch after missing-key delete\n"); + pass = 0; + } + +done: + BULKI_free(bulki, 1); + FUNC_LEAVE(pass); +} + int bulki_small_json_serialization_test() { @@ -364,6 +583,12 @@ main(int argc, char *argv[]) LOG_INFO("test_base_array_entitiy RST = %d\n", test_base_array_entitiy()); LOG_INFO("test_embedded_entitiy RST = %d\n", test_embedded_entitiy()); LOG_INFO("test_nested_entitiy RST = %d\n", test_bulki_in_entitiy()); + LOG_INFO("test_incremental_array_append_sizes RST = %d\n", test_incremental_array_append_sizes()); + LOG_INFO("test_preallocated_array_roundtrip RST = %d\n", test_preallocated_array_roundtrip()); + LOG_INFO("test_incremental_bulki_put_total_size RST = %d\n", test_incremental_bulki_put_total_size()); + LOG_INFO("test_incremental_entity_array_append_sizes RST = %d\n", + test_incremental_entity_array_append_sizes()); + LOG_INFO("test_incremental_bulki_delete_total_size RST = %d\n", test_incremental_bulki_delete_total_size()); LOG_INFO("bulki_small_json_serialization_test RST = %d\n", bulki_small_json_serialization_test()); FUNC_LEAVE(0); diff --git a/src/commons/serde/bulki/include/bulki.h b/src/commons/serde/bulki/include/bulki.h index ac85fe59e..b53625771 100644 --- a/src/commons/serde/bulki/include/bulki.h +++ b/src/commons/serde/bulki/include/bulki.h @@ -19,6 +19,7 @@ typedef struct { pdc_c_var_class_t pdc_class; /**< Class of the value */ pdc_c_var_type_t pdc_type; /**< Data type of the value */ uint64_t count; /**< Number of elements in the array */ + uint64_t capacity; /**< Allocated slots (array types only; not serialized) */ uint64_t size; // size in byte of the data. void * data; /**< Pointer to the value data */ } BULKI_Entity; @@ -88,12 +89,34 @@ BULKI_Entity *BULKI_ENTITY(void *data, uint64_t count, pdc_c_var_type_t pdc_type */ BULKI_Entity *empty_Bent_Array_Entity(); +/** + * @brief create an empty array of BULKI_Entity with pre-allocated capacity + * + * Pre-allocates `initial_capacity` slots so `_incremental` appends avoid repeated + * realloc. Pair with `BULKI_ENTITY_append_BULKI_Entity_incremental`. + * + * @param initial_capacity Number of BULKI_Entity slots to pre-allocate; use 0 for no pre-allocation + * @return Pointer to the created BULKI_Entity structure + */ +BULKI_Entity *empty_Bent_Array_Entity_with_capacity(int initial_capacity); + /** * @brief create an empty BULKI structure, usually used as a wrapper for a BULKI * @return Pointer to the created BULKI_Entity structure */ BULKI_Entity *empty_BULKI_Array_Entity(); +/** + * @brief create an empty array of BULKI with pre-allocated capacity (similar to BULKI_init) + * + * Pre-allocates `initial_capacity` slots so `_incremental` appends avoid repeated + * realloc. Pair with `BULKI_ENTITY_append_BULKI_incremental`. + * + * @param initial_capacity Number of BULKI slots to pre-allocate; use 0 for no pre-allocation + * @return Pointer to the created BULKI_Entity structure + */ +BULKI_Entity *empty_BULKI_Array_Entity_with_capacity(int initial_capacity); + /** * @brief Create a BULKI_Entity structure with data of base type, which is a wrapper of BULKI_ENTITY((void * *)single_data, 1, pdc_type, PDC_CLS_ITEM) @@ -117,6 +140,11 @@ BULKI_Entity *BULKI_array_ENTITY(void *data, uint64_t count, pdc_c_var_type_t pd /** * @brief Append a BULKI_Entity structure to the BULKI_Entity structure + * + * Recomputes the full serialized size of `dest` via `get_BULKI_Entity_size()` after + * each append. For bulk construction, prefer `BULKI_ENTITY_append_BULKI_Entity_incremental` + * with `empty_Bent_Array_Entity_with_capacity`. + * * You need to make sure the content the src structure is the final version before calling this function. * Any change to the content of src after calling this function will not be reflected in the serialized data * structure. @@ -130,9 +158,28 @@ BULKI_Entity *BULKI_array_ENTITY(void *data, uint64_t count, pdc_c_var_type_t pd */ BULKI_Entity *BULKI_ENTITY_append_BULKI_Entity(BULKI_Entity *dest, BULKI_Entity *src); +/** + * @brief Append a BULKI_Entity to an array without recomputing the full subtree size. + * + * Updates `dest->size` and `dest->count` in O(1) using the cached wire size of `src`. + * Uses pre-allocated capacity when available (see `empty_Bent_Array_Entity_with_capacity`). + * Same ownership and content-finalization rules as `BULKI_ENTITY_append_BULKI_Entity`. + * + * @param dest Destination array of BULKI_Entity (PDC_CLS_ARRAY, PDC_BULKI_ENT) + * @param src Source entity to append; its content must be final before this call + * @return Pointer to `dest`, or NULL on error + */ +BULKI_Entity *BULKI_ENTITY_append_BULKI_Entity_incremental(BULKI_Entity *dest, BULKI_Entity *src); + /** * @brief Append a BULKI structure to the BULKI_Entity structure that are returned by - * `empty_BULKI_Array_Entity` call. You need to make sure the content the src structure is the final version + * `empty_BULKI_Array_Entity` call. + * + * Recomputes the full serialized size of `dest` via `get_BULKI_Entity_size()` after + * each append. For bulk construction, prefer `BULKI_ENTITY_append_BULKI_incremental` + * with `empty_BULKI_Array_Entity_with_capacity`. + * + * You need to make sure the content the src structure is the final version * before calling this function. Any change to the content of src after calling this function will not be * reflected in the serialized data structure. If you need to change the content of src after calling this * function, you need to iterate through all the BULKI structure in the dest structure and update them. We do @@ -144,6 +191,21 @@ BULKI_Entity *BULKI_ENTITY_append_BULKI_Entity(BULKI_Entity *dest, BULKI_Entity */ BULKI_Entity *BULKI_ENTITY_append_BULKI(BULKI_Entity *dest, BULKI *src); +/** + * @brief Append a BULKI map to an array without recomputing the full subtree size. + * + * Updates `dest->size` and `dest->count` in O(1) using the cached wire size of `src` + * (`src->totalSize`). Uses pre-allocated capacity when available (see + * `empty_BULKI_Array_Entity_with_capacity`). Prefer building `src` with + * `BULKI_put_incremental` so `totalSize` stays current. Same ownership and + * content-finalization rules as `BULKI_ENTITY_append_BULKI`. + * + * @param dest Destination array of BULKI (PDC_CLS_ARRAY, PDC_BULKI) + * @param src Source BULKI map to append; its content must be final before this call + * @return Pointer to `dest`, or NULL on error + */ +BULKI_Entity *BULKI_ENTITY_append_BULKI_incremental(BULKI_Entity *dest, BULKI *src); + /* ================--------- BULKI_Entity Data Retrieval APIs -----------================ */ /** @@ -255,6 +317,10 @@ BULKI *BULKI_init(int initial_field_count); /** * @brief Put a key-value pair to the serialized data structure. If the key already exists, update the value. + * + * Recomputes `totalSize` via `get_BULKI_size()` after each put. For bulk tree + * construction, prefer `BULKI_put_incremental`. + * * You need to make sure the content in both key and value are the final version before calling this function. * Any change to the key or value after calling this function will not be reflected in the serialized data * structure. @@ -266,6 +332,21 @@ BULKI *BULKI_init(int initial_field_count); */ void BULKI_put(BULKI *bulki, BULKI_Entity *key, BULKI_Entity *value); +/** + * @brief Put a key-value pair and refresh `totalSize` incrementally in O(1). + * + * Maintains `headerSize`, `dataSize`, and `totalSize` from cached entity sizes + * instead of calling `get_BULKI_size()`. Use this when building large BULKI trees + * (e.g. checkpoints). Do not mix with `BULKI_put` on the same structure unless + * you accept a full recompute on the next `BULKI_put` call. Same key/value + * finalization rules as `BULKI_put`. + * + * @param bulki Pointer to the BULKI structure + * @param key Pointer to the key entity + * @param value Pointer to the value entity + */ +void BULKI_put_incremental(BULKI *bulki, BULKI_Entity *key, BULKI_Entity *value); + /** * @brief Delete a key-value pair from the serialized data structure * @@ -275,6 +356,19 @@ void BULKI_put(BULKI *bulki, BULKI_Entity *key, BULKI_Entity *value); */ BULKI_Entity *BULKI_delete(BULKI *bulki, BULKI_Entity *key); +/** + * @brief Delete a key-value pair and refresh `totalSize` incrementally in O(1). + * + * Same behavior as `BULKI_delete`, but updates `totalSize` via cached header/data + * sizes instead of calling `get_BULKI_size()`. Pair with `BULKI_put_incremental` + * when maintaining structures built on the incremental path. + * + * @param bulki Pointer to the BULKI structure + * @param key Pointer to the key entity to remove + * @return Pointer to the removed value entity, or NULL if the key was not found + */ +BULKI_Entity *BULKI_delete_incremental(BULKI *bulki, BULKI_Entity *key); + /* ================--------- BULKI Data Retrieval APIs -----------================ */ /** diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 2ec1f06ec..17e78ecbb 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1239,13 +1239,13 @@ PDC_Server_checkpoint() checkpoint_bulki = BULKI_init(5); // BULKI version number for validation - BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING), + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING), BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); // checkpoint containers n_entry = hash_table_num_entries(container_hash_table_g); - BULKI_Entity *containers_array = empty_BULKI_Array_Entity(); + BULKI_Entity *containers_array = empty_BULKI_Array_Entity_with_capacity(n_entry > 0 ? n_entry : 1); hash_table_iterate(container_hash_table_g, &hash_table_iter); while (n_entry != 0 && hash_table_iter_has_more(&hash_table_iter)) { @@ -1255,22 +1255,23 @@ PDC_Server_checkpoint() BULKI *container_entry = BULKI_init(2); hash_key = PDC_get_hash_by_name(cont_head->cont_name); - BULKI_put(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put_incremental(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // store the container structure as binary blob - BULKI_put(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_put_incremental(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); - BULKI_ENTITY_append_BULKI(containers_array, container_entry); + BULKI_ENTITY_append_BULKI_incremental(containers_array, container_entry); } - BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); - BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity(); + BULKI_Entity *metadata_entries_array = + empty_BULKI_Array_Entity_with_capacity(n_entry > 0 ? n_entry : 1); hash_table_iterate(metadata_hash_table_g, &hash_table_iter); @@ -1281,15 +1282,16 @@ PDC_Server_checkpoint() BULKI *hash_entry = BULKI_init(3); // Store number of objects - BULKI_put(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING), + BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING), BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); hash_key = PDC_get_hash_by_name(head->metadata->obj_name); - BULKI_put(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), + BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // array of metadata objects - BULKI_Entity *metadata_objs_array = empty_BULKI_Array_Entity(); + BULKI_Entity *metadata_objs_array = + empty_BULKI_Array_Entity_with_capacity(head->n_obj > 0 ? head->n_obj : 1); // iterate every metadata structure in current entry DL_FOREACH(head->metadata, elt) @@ -1297,40 +1299,40 @@ PDC_Server_checkpoint() BULKI *metadata_obj = BULKI_init(3); // store metadata structure - BULKI_put(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), + BULKI_put_incremental(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); // kv tags DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); - BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity(); + BULKI_Entity *kvtags_array = empty_BULKI_Array_Entity_with_capacity(n_kvtag > 0 ? n_kvtag : 1); DL_FOREACH(elt->kvtag_list_head, kvlist_elt) { BULKI *kvtag_entry = BULKI_init(4); - BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), + BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); - BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING), + BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING), + BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING), BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); - BULKI_put(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING), + BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING), BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, PDC_UINT8, PDC_CLS_ARRAY)); - BULKI_ENTITY_append_BULKI(kvtags_array, kvtag_entry); + BULKI_ENTITY_append_BULKI_incremental(kvtags_array, kvtag_entry); } - BULKI_put(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); + BULKI_put_incremental(metadata_obj, BULKI_singleton_ENTITY("kvtags", PDC_STRING), kvtags_array); // storage regions n_region = 0; DL_COUNT(elt->storage_region_list_head, region_elt, n_region); - BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); + BULKI_Entity *regions_array = empty_BULKI_Array_Entity_with_capacity(n_region > 0 ? n_region : 1); if (n_region > 0) { n_write_region = 0; DL_FOREACH(elt->storage_region_list_head, region_elt) @@ -1338,7 +1340,7 @@ PDC_Server_checkpoint() BULKI *region_entry = BULKI_init(3); // store region structure - BULKI_put(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); // store histogram if present @@ -1346,34 +1348,34 @@ PDC_Server_checkpoint() if (region_elt->region_hist != NULL) has_hist = 1; - BULKI_put(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING), + BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING), BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); if (has_hist == 1) { BULKI *histogram = BULKI_init(5); - BULKI_put(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put(histogram, BULKI_singleton_ENTITY("range", PDC_STRING), + BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("range", PDC_STRING), BULKI_ENTITY(region_elt->region_hist->range, region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); - BULKI_put(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING), + BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING), BULKI_ENTITY(region_elt->region_hist->bin, region_elt->region_hist->nbin, PDC_UINT64, PDC_CLS_ARRAY)); - BULKI_put(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); - BULKI_put(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING), + BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING), BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } - BULKI_ENTITY_append_BULKI(regions_array, region_entry); + BULKI_ENTITY_append_BULKI_incremental(regions_array, region_entry); n_write_region++; } @@ -1381,53 +1383,54 @@ PDC_Server_checkpoint() LOG_ERROR("Error with number of regions\n"); } - BULKI_put(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); + BULKI_put_incremental(metadata_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); - BULKI_ENTITY_append_BULKI(metadata_objs_array, metadata_obj); + BULKI_ENTITY_append_BULKI_incremental(metadata_objs_array, metadata_obj); metadata_size++; region_count += n_region; } // end for metadata entry linked list - BULKI_put(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); + BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); - BULKI_ENTITY_append_BULKI(metadata_entries_array, hash_entry); + BULKI_ENTITY_append_BULKI_incremental(metadata_entries_array, hash_entry); } // end for hash table metadata entry - BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), metadata_entries_array); // data server regions data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); - BULKI_Entity *dataserver_regions_array = empty_BULKI_Array_Entity(); + BULKI_Entity *dataserver_regions_array = + empty_BULKI_Array_Entity_with_capacity(n_objs > 0 ? n_objs : 1); DL_FOREACH(dataserver_region_g, region) { BULKI *dataserver_obj = BULKI_init(2); - BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), + BULKI_put_incremental(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); DL_COUNT(region->region_storage_head, region_elt, n_region); - BULKI_Entity *ds_regions_array = empty_BULKI_Array_Entity(); + BULKI_Entity *ds_regions_array = empty_BULKI_Array_Entity_with_capacity(n_region > 0 ? n_region : 1); DL_FOREACH(region->region_storage_head, region_elt) { BULKI *ds_region = BULKI_init(1); - BULKI_put(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_put_incremental(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING), BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); - BULKI_ENTITY_append_BULKI(ds_regions_array, ds_region); + BULKI_ENTITY_append_BULKI_incremental(ds_regions_array, ds_region); } - BULKI_put(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), ds_regions_array); + BULKI_put_incremental(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), ds_regions_array); - BULKI_ENTITY_append_BULKI(dataserver_regions_array, dataserver_obj); + BULKI_ENTITY_append_BULKI_incremental(dataserver_regions_array, dataserver_obj); } - BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), dataserver_regions_array); // transfer request metadata query @@ -1438,7 +1441,7 @@ PDC_Server_checkpoint() PGOTO_ERROR(FAIL, "Transfer query checkpoint failed"); } - BULKI_put(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index 440aa763c..7b01abc0a 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -307,44 +307,56 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) pthread_mutex_lock(&metadata_query_mutex); + obj_temp = metadata_server_objs; + while (obj_temp) { + obj_count++; + obj_temp = obj_temp->next; + } + bulki = BULKI_init(1); - BULKI_Entity *objects_array = empty_BULKI_Array_Entity(); + BULKI_Entity *objects_array = empty_BULKI_Array_Entity_with_capacity(obj_count > 0 ? obj_count : 1); obj_temp = metadata_server_objs; while (obj_temp) { BULKI *obj_bulki = BULKI_init(3); + int reg_count = 0; - BULKI_put(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING), + region_temp = obj_temp->regions; + while (region_temp) { + reg_count++; + region_temp = region_temp->next; + } + + BULKI_put_incremental(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING), BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); - BULKI_put(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING), + BULKI_put_incremental(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING), BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_Entity *regions_array = empty_BULKI_Array_Entity(); + BULKI_Entity *regions_array = empty_BULKI_Array_Entity_with_capacity(reg_count > 0 ? reg_count : 1); region_temp = obj_temp->regions; while (region_temp) { BULKI *region_bulki = BULKI_init(2); - BULKI_put(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING), + BULKI_put_incremental(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING), BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_put_incremental(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, PDC_UINT64, PDC_CLS_ARRAY)); - BULKI_ENTITY_append_BULKI(regions_array, region_bulki); + BULKI_ENTITY_append_BULKI_incremental(regions_array, region_bulki); region_temp = region_temp->next; } - BULKI_put(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); + BULKI_put_incremental(obj_bulki, BULKI_singleton_ENTITY("regions", PDC_STRING), regions_array); - BULKI_ENTITY_append_BULKI(objects_array, obj_bulki); + BULKI_ENTITY_append_BULKI_incremental(objects_array, obj_bulki); - obj_count++; obj_temp = obj_temp->next; } - BULKI_put(bulki, BULKI_singleton_ENTITY("objects", PDC_STRING), objects_array); + BULKI_put_incremental(bulki, BULKI_singleton_ENTITY("objects", PDC_STRING), objects_array); pthread_mutex_unlock(&metadata_query_mutex); From 8d714d60c70c5a1bb7e16b1b086dc3c262f793f0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 16 Jun 2026 18:03:36 +0000 Subject: [PATCH 16/30] Committing clang-format changes --- src/commons/serde/bulki/bulki.c | 10 +-- src/commons/serde/bulki/bulki_serde.c | 18 ++-- src/commons/serde/bulki/bulki_serde_test.c | 15 ++-- src/server/pdc_server.c | 84 ++++++++++--------- ...dc_server_region_transfer_metadata_query.c | 11 +-- 5 files changed, 76 insertions(+), 62 deletions(-) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index 77c20d007..d8966e55f 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -7,7 +7,7 @@ /** Wire size of a BULKI_Entity header: pdc_class, pdc_type, count, size fields. */ #define BULKI_ENTITY_WIRE_HEADER_SIZE ((size_t)(sizeof(int8_t) * 2 + sizeof(uint64_t) * 2)) /** Wire size of a BULKI map header: totalSize, numKeys, headerSize, dataSize, offsets. */ -#define BULKI_WIRE_META_SIZE ((size_t)(sizeof(uint64_t) * 6)) +#define BULKI_WIRE_META_SIZE ((size_t)(sizeof(uint64_t) * 6)) static size_t bulki_entity_wire_size(BULKI_Entity *entity); static size_t bulki_wire_size(BULKI *bulki); @@ -315,10 +315,10 @@ BULKI_ENTITY_append_BULKI_incremental(BULKI_Entity *dest, BULKI *src) FUNC_LEAVE(NULL); } - child_size = bulki_wire_size(src); + child_size = bulki_wire_size(src); bulki_entity_array_ensure_capacity(dest, sizeof(BULKI)); memcpy((BULKI *)dest->data + dest->count, src, sizeof(BULKI)); - dest->size += child_size; + dest->size += child_size; dest->count++; FUNC_LEAVE(dest); @@ -383,10 +383,10 @@ BULKI_ENTITY_append_BULKI_Entity_incremental(BULKI_Entity *dest, BULKI_Entity *s FUNC_LEAVE(NULL); } - child_size = bulki_entity_wire_size(src); + child_size = bulki_entity_wire_size(src); bulki_entity_array_ensure_capacity(dest, sizeof(BULKI_Entity)); memcpy((BULKI_Entity *)dest->data + dest->count, src, sizeof(BULKI_Entity)); - dest->size += child_size; + dest->size += child_size; dest->count++; FUNC_LEAVE(dest); diff --git a/src/commons/serde/bulki/bulki_serde.c b/src/commons/serde/bulki/bulki_serde.c index 2b03e00db..3259978b9 100644 --- a/src/commons/serde/bulki/bulki_serde.c +++ b/src/commons/serde/bulki/bulki_serde.c @@ -68,7 +68,8 @@ BULKI_Entity_serialize_to_buffer(BULKI_Entity *entity, void *buffer, size_t *off FUNC_ENTER(NULL); // serialize the size - // note: redundant tree walk — entity->size is already cached after incremental build; could use it directly + // note: redundant tree walk — entity->size is already cached after incremental build; could use it + // directly uint64_t size = (uint64_t)get_BULKI_Entity_size(entity); *offset += BULKI_vle_encode_uint(size, buffer + *offset); @@ -123,7 +124,8 @@ BULKI_Entity_serialize(BULKI_Entity *entity, size_t *size) { FUNC_ENTER(NULL); - // note: redundant tree walk — entity->size is already cached; serialize_to_buffer recomputes per entity below + // note: redundant tree walk — entity->size is already cached; serialize_to_buffer recomputes per entity + // below size_t estimated_size = get_BULKI_Entity_size(entity); void * buffer = PDC_calloc(1, estimated_size); size_t offset = 0; @@ -204,8 +206,9 @@ BULKI_Entity_serialize_to_file(BULKI_Entity *entity, FILE *fp) FUNC_ENTER(NULL); size_t size; - // note: buffer-then-fwrite — builds entire entity in memory before a single fwrite; consider streaming to fp - void * buffer = BULKI_Entity_serialize(entity, &size); + // note: buffer-then-fwrite — builds entire entity in memory before a single fwrite; consider streaming to + // fp + void *buffer = BULKI_Entity_serialize(entity, &size); fwrite(buffer, size, 1, fp); buffer = (void *)PDC_free(buffer); fclose(fp); @@ -218,9 +221,10 @@ BULKI_serialize_to_file(BULKI *bulki, FILE *fp) { FUNC_ENTER(NULL); - size_t size = 0; - // note: buffer-then-fwrite — full checkpoint serialized to RAM first; consider streaming serialize_to_file - void * buffer = BULKI_serialize(bulki, &size); + size_t size = 0; + // note: buffer-then-fwrite — full checkpoint serialized to RAM first; consider streaming + // serialize_to_file + void *buffer = BULKI_serialize(bulki, &size); fwrite(buffer, size, 1, fp); buffer = (void *)PDC_free(buffer); fclose(fp); diff --git a/src/commons/serde/bulki/bulki_serde_test.c b/src/commons/serde/bulki/bulki_serde_test.c index 59827c278..76b0350c3 100644 --- a/src/commons/serde/bulki/bulki_serde_test.c +++ b/src/commons/serde/bulki/bulki_serde_test.c @@ -338,19 +338,19 @@ test_preallocated_array_roundtrip() int val = i * 10; BULKI_put_incremental(item, BULKI_singleton_ENTITY("name", PDC_STRING), - BULKI_singleton_ENTITY("entry", PDC_STRING)); + BULKI_singleton_ENTITY("entry", PDC_STRING)); BULKI_put_incremental(item, BULKI_singleton_ENTITY("value", PDC_STRING), - BULKI_ENTITY(&val, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_ENTITY(&val, 1, PDC_INT, PDC_CLS_ITEM)); BULKI_ENTITY_append_BULKI_incremental(arr, item); } BULKI_put_incremental(root, BULKI_singleton_ENTITY("entries", PDC_STRING), arr); { - size_t size; - void * buffer = BULKI_serialize(root, &size); - BULKI * deserialized = BULKI_deserialize(buffer); - int equal = BULKI_equal(root, deserialized); + size_t size; + void * buffer = BULKI_serialize(root, &size); + BULKI *deserialized = BULKI_deserialize(buffer); + int equal = BULKI_equal(root, deserialized); LOG_INFO("preallocated array roundtrip equal: %d\n", equal); BULKI_free(deserialized, 1); @@ -588,7 +588,8 @@ main(int argc, char *argv[]) LOG_INFO("test_incremental_bulki_put_total_size RST = %d\n", test_incremental_bulki_put_total_size()); LOG_INFO("test_incremental_entity_array_append_sizes RST = %d\n", test_incremental_entity_array_append_sizes()); - LOG_INFO("test_incremental_bulki_delete_total_size RST = %d\n", test_incremental_bulki_delete_total_size()); + LOG_INFO("test_incremental_bulki_delete_total_size RST = %d\n", + test_incremental_bulki_delete_total_size()); LOG_INFO("bulki_small_json_serialization_test RST = %d\n", bulki_small_json_serialization_test()); FUNC_LEAVE(0); diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 17e78ecbb..cea3db0df 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1240,7 +1240,7 @@ PDC_Server_checkpoint() // BULKI version number for validation BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING), - BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); + BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); // checkpoint containers n_entry = hash_table_num_entries(container_hash_table_g); @@ -1256,22 +1256,23 @@ PDC_Server_checkpoint() hash_key = PDC_get_hash_by_name(cont_head->cont_name); BULKI_put_incremental(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), - BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // store the container structure as binary blob - BULKI_put_incremental(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), - BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_put_incremental( + container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING), + BULKI_ENTITY(cont_head, sizeof(pdc_cont_hash_table_entry_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI_incremental(containers_array, container_entry); } - BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), containers_array); + BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("containers", PDC_STRING), + containers_array); // checkpoint metadata hash table n_entry = hash_table_num_entries(metadata_hash_table_g); - BULKI_Entity *metadata_entries_array = - empty_BULKI_Array_Entity_with_capacity(n_entry > 0 ? n_entry : 1); + BULKI_Entity *metadata_entries_array = empty_BULKI_Array_Entity_with_capacity(n_entry > 0 ? n_entry : 1); hash_table_iterate(metadata_hash_table_g, &hash_table_iter); @@ -1283,11 +1284,11 @@ PDC_Server_checkpoint() // Store number of objects BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING), - BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_ENTITY(&head->n_obj, 1, PDC_INT, PDC_CLS_ITEM)); hash_key = PDC_get_hash_by_name(head->metadata->obj_name); BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING), - BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); + BULKI_ENTITY(&hash_key, 1, PDC_UINT32, PDC_CLS_ITEM)); // array of metadata objects BULKI_Entity *metadata_objs_array = @@ -1300,7 +1301,7 @@ PDC_Server_checkpoint() // store metadata structure BULKI_put_incremental(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING), - BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_ENTITY(elt, sizeof(pdc_metadata_t), PDC_UINT8, PDC_CLS_ARRAY)); // kv tags DL_COUNT(elt->kvtag_list_head, kvlist_elt, n_kvtag); @@ -1311,17 +1312,17 @@ PDC_Server_checkpoint() BULKI *kvtag_entry = BULKI_init(4); BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING), - BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); + BULKI_singleton_ENTITY(kvlist_elt->kvtag->name, PDC_STRING)); BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING), - BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); + BULKI_ENTITY(&kvlist_elt->kvtag->size, 1, PDC_UINT32, PDC_CLS_ITEM)); BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING), - BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); + BULKI_ENTITY(&kvlist_elt->kvtag->type, 1, PDC_INT8, PDC_CLS_ITEM)); BULKI_put_incremental(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING), - BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, PDC_UINT8, - PDC_CLS_ARRAY)); + BULKI_ENTITY(kvlist_elt->kvtag->value, kvlist_elt->kvtag->size, + PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI_incremental(kvtags_array, kvtag_entry); } @@ -1340,8 +1341,9 @@ PDC_Server_checkpoint() BULKI *region_entry = BULKI_init(3); // store region structure - BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), - BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_put_incremental( + region_entry, BULKI_singleton_ENTITY("region", PDC_STRING), + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); // store histogram if present int has_hist = 0; @@ -1349,30 +1351,35 @@ PDC_Server_checkpoint() has_hist = 1; BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING), - BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_ENTITY(&has_hist, 1, PDC_INT, PDC_CLS_ITEM)); if (has_hist == 1) { BULKI *histogram = BULKI_init(5); - BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), - BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_put_incremental( + histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->dtype, 1, PDC_INT, PDC_CLS_ITEM)); - BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), - BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_put_incremental( + histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->nbin, 1, PDC_INT, PDC_CLS_ITEM)); BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("range", PDC_STRING), - BULKI_ENTITY(region_elt->region_hist->range, - region_elt->region_hist->nbin * 2, PDC_DOUBLE, PDC_CLS_ARRAY)); + BULKI_ENTITY(region_elt->region_hist->range, + region_elt->region_hist->nbin * 2, PDC_DOUBLE, + PDC_CLS_ARRAY)); BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING), - BULKI_ENTITY(region_elt->region_hist->bin, region_elt->region_hist->nbin, - PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_ENTITY(region_elt->region_hist->bin, + region_elt->region_hist->nbin, PDC_UINT64, + PDC_CLS_ARRAY)); - BULKI_put_incremental(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), - BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); + BULKI_put_incremental( + histogram, BULKI_singleton_ENTITY("incr", PDC_STRING), + BULKI_ENTITY(®ion_elt->region_hist->incr, 1, PDC_DOUBLE, PDC_CLS_ITEM)); BULKI_put_incremental(region_entry, BULKI_singleton_ENTITY("histogram", PDC_STRING), - BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); + BULKI_ENTITY(histogram, 1, PDC_BULKI, PDC_CLS_ITEM)); } BULKI_ENTITY_append_BULKI_incremental(regions_array, region_entry); @@ -1391,27 +1398,27 @@ PDC_Server_checkpoint() region_count += n_region; } // end for metadata entry linked list - BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), metadata_objs_array); + BULKI_put_incremental(hash_entry, BULKI_singleton_ENTITY("metadata_objects", PDC_STRING), + metadata_objs_array); BULKI_ENTITY_append_BULKI_incremental(metadata_entries_array, hash_entry); } // end for hash table metadata entry BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("metadata_entries", PDC_STRING), - metadata_entries_array); + metadata_entries_array); // data server regions data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); - BULKI_Entity *dataserver_regions_array = - empty_BULKI_Array_Entity_with_capacity(n_objs > 0 ? n_objs : 1); + BULKI_Entity *dataserver_regions_array = empty_BULKI_Array_Entity_with_capacity(n_objs > 0 ? n_objs : 1); DL_FOREACH(dataserver_region_g, region) { BULKI *dataserver_obj = BULKI_init(2); BULKI_put_incremental(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING), - BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + BULKI_ENTITY(®ion->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); DL_COUNT(region->region_storage_head, region_elt, n_region); @@ -1420,18 +1427,19 @@ PDC_Server_checkpoint() { BULKI *ds_region = BULKI_init(1); BULKI_put_incremental(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING), - BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); + BULKI_ENTITY(region_elt, sizeof(region_list_t), PDC_UINT8, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI_incremental(ds_regions_array, ds_region); } - BULKI_put_incremental(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), ds_regions_array); + BULKI_put_incremental(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING), + ds_regions_array); BULKI_ENTITY_append_BULKI_incremental(dataserver_regions_array, dataserver_obj); } BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("dataserver_regions", PDC_STRING), - dataserver_regions_array); + dataserver_regions_array); // transfer request metadata query BULKI *transfer_query_bulki = NULL; @@ -1442,7 +1450,7 @@ PDC_Server_checkpoint() } BULKI_put_incremental(checkpoint_bulki, BULKI_singleton_ENTITY("transfer_query", PDC_STRING), - BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); + BULKI_ENTITY(transfer_query_bulki, 1, PDC_BULKI, PDC_CLS_ITEM)); // ========== Serialize and Write ========== if (use_tmpfs) diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index 7b01abc0a..802ba0539 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -328,10 +328,10 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) } BULKI_put_incremental(obj_bulki, BULKI_singleton_ENTITY("obj_id", PDC_STRING), - BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); + BULKI_ENTITY(&obj_temp->obj_id, 1, PDC_UINT64, PDC_CLS_ITEM)); BULKI_put_incremental(obj_bulki, BULKI_singleton_ENTITY("ndim", PDC_STRING), - BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); + BULKI_ENTITY(&obj_temp->ndim, 1, PDC_INT, PDC_CLS_ITEM)); BULKI_Entity *regions_array = empty_BULKI_Array_Entity_with_capacity(reg_count > 0 ? reg_count : 1); @@ -340,10 +340,11 @@ transfer_request_metadata_query_checkpoint_bulki(BULKI **checkpoint_bulki) BULKI *region_bulki = BULKI_init(2); BULKI_put_incremental(region_bulki, BULKI_singleton_ENTITY("data_server_id", PDC_STRING), - BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); + BULKI_ENTITY(®ion_temp->data_server_id, 1, PDC_UINT32, PDC_CLS_ITEM)); - BULKI_put_incremental(region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), - BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, PDC_UINT64, PDC_CLS_ARRAY)); + BULKI_put_incremental( + region_bulki, BULKI_singleton_ENTITY("reg_offset_size", PDC_STRING), + BULKI_ENTITY(region_temp->reg_offset, obj_temp->ndim * 2, PDC_UINT64, PDC_CLS_ARRAY)); BULKI_ENTITY_append_BULKI_incremental(regions_array, region_bulki); region_temp = region_temp->next; From 761262037144b2dbde1784eab2c5f5e116e5b671 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Tue, 16 Jun 2026 15:50:50 -0400 Subject: [PATCH 17/30] Add parameterized checkpoint/restart test runners and CTest coverage Introduce serial and MPI scripts that start pdc_server, run a test program with caller-supplied arguments, close the server (triggering checkpoint), then repeat with restart for subsequent programs. Wire kvtag_add_scale and kvtag_get_verify_scale into the default test build and register serial and MPI CTest entries that exercise add/verify across server close and restart (1000 objects, 100 tags). --- src/tests/CMakeLists.txt | 12 +++- ...kpoint_restart_mpi_test_with_prog_param.sh | 67 +++++++++++++++++++ ...checkpoint_restart_test_with_prog_param.sh | 59 ++++++++++++++++ 3 files changed, 135 insertions(+), 3 deletions(-) create mode 100755 src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh create mode 100755 src/tests/run_checkpoint_restart_test_with_prog_param.sh diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 574d84d9f..0a1743006 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,6 +102,8 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall + tags/kvtag_add_scale + tags/kvtag_get_verify_scale ) # TODO: Check if import_vpic.c is needed. If yes, we have to add the following : @@ -136,8 +138,6 @@ if(BUILD_MPI_TESTING) tags/kvtag_affix_query_scale tags/kvtag_add_get_benchmark tags/kvtag_add_get_scale - tags/kvtag_add_scale - tags/kvtag_get_verify_scale ) if(PDC_ENABLE_IDIOMS) @@ -218,6 +218,8 @@ set(SCRIPTS run_multiple_test.sh run_multiple_mpi_test.sh run_checkpoint_restart_test.sh + run_checkpoint_restart_test_with_prog_param.sh + run_checkpoint_restart_mpi_test_with_prog_param.sh ) foreach(script ${SCRIPTS}) @@ -318,7 +320,8 @@ add_test(NAME read_obj_int64 WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTOR add_test(NAME read_obj_uint64 WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./read_obj o 1 uint64) add_test(NAME read_obj_int16 WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./read_obj o 1 int16) add_test(NAME read_obj_int8 WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./read_obj o 1 int8) -add_test(NAME vpicio_bdcats_transfer_request WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_test.sh ./vpicio ./bdcats) +add_test(NAME vpicio_bdcats_transfer_request WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_test.sh ./vpicio ./bdcats) +add_test(NAME kvtag_checkpoint_restart WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_test_with_prog_param.sh "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") set_tests_properties(pdc_init PROPERTIES LABELS serial) @@ -391,6 +394,7 @@ set_tests_properties(read_obj_uint64 PROPERTIES LABELS serial) set_tests_properties(read_obj_int16 PROPERTIES LABELS serial) set_tests_properties(read_obj_int8 PROPERTIES LABELS serial) set_tests_properties(vpicio_bdcats_transfer_request PROPERTIES LABELS serial) +set_tests_properties(kvtag_checkpoint_restart PROPERTIES LABELS serial) if(BUILD_MPI_TESTING) add_test(NAME read_obj_shared_int WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./read_obj_shared ${MPI_RUN_CMD} 4 6 o 1 int) @@ -463,6 +467,7 @@ if(BUILD_MPI_TESTING) add_test(NAME obj_get_data_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./obj_get_data ${MPI_RUN_CMD} 4 6 ) add_test(NAME vpicio_bdcats_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_mpi_test.sh ${MPI_RUN_CMD} 4 6 ./vpicio ./bdcats) add_test(NAME vpicio_mpi_small_cache WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./vpicio ${MPI_RUN_CMD} 1 16) + add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") set_tests_properties(read_obj_shared_int PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(read_obj_shared_float PROPERTIES LABELS "parallel;parallel_obj" ) @@ -534,4 +539,5 @@ if(BUILD_MPI_TESTING) set_tests_properties(obj_info_mpi PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(obj_put_data_mpi PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(obj_get_data_mpi PROPERTIES LABELS "parallel;parallel_obj" ) + set_tests_properties(kvtag_checkpoint_restart_mpi PROPERTIES LABELS "parallel;parallel_checkpoint" ) endif() diff --git a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh new file mode 100755 index 000000000..f4da849e3 --- /dev/null +++ b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# MPI version of the checkpoint/restart test runner with program parameters. +# We assume that if the library build has enabled MPI, that LD_LIBRARY_PATH is +# defined and points to the MPI libraries used by the linker (e.g. -L -lmpi) + +# Usage: +# ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ +# " [args...]" " [args...]" ... +# +# Example: +# ./run_checkpoint_restart_mpi_test_with_prog_param.sh srun 2 4 \ +# "/path/to/kvtag_add_scale 1000 100" \ +# "/path/to/kvtag_get_verify_scale 1000 100" + +extra_cmd="" + +if [[ "$SUPERCOMPUTER" == "perlmutter" ]]; then + extra_cmd="--mem=25600 --cpu_bind=cores --overlap" +fi + +if [ $# -lt 4 ]; then + echo "Usage: $0 \" [args...]\" ..." + exit -1 +fi + +mpi_cmd="$1" +n_servers="$2" +n_client="$3" +shift 3 + +rm -rf pdc_tmp pdc_data + +restart=" " +ret=0 +for test_args in "$@" +do + # Split the quoted string into the executable and its parameters + read -ra cmd_parts <<< "$test_args" + test_exe="${cmd_parts[0]}" + test_params="${cmd_parts[@]:1}" + + if [ ! -x "$test_exe" ]; then + echo "test: $test_exe not found or not an executable" && exit -2 + fi + + # START the server (in the background) + echo "$mpi_cmd -n $n_servers -c 2 $extra_cmd ./pdc_server $restart &" + $mpi_cmd -n $n_servers -c 2 $extra_cmd "./pdc_server" $restart & + + # WAIT a bit... + sleep 1 + + # RUN the actual test + echo "testing: $test_exe $test_params" + echo "$mpi_cmd -n $n_client -c 2 $extra_cmd $test_exe $test_params" + $mpi_cmd -n $n_client -c 2 $extra_cmd $test_exe $test_params + ret="$?" + + # Shutdown the server before next iteration + echo "Close server" + echo "$mpi_cmd -n $n_servers -c 2 $extra_cmd ./close_server" + $mpi_cmd -n $n_servers -c 2 $extra_cmd "./close_server" + + restart="restart" +done + +exit $ret \ No newline at end of file diff --git a/src/tests/run_checkpoint_restart_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_test_with_prog_param.sh new file mode 100755 index 000000000..2da3c58e0 --- /dev/null +++ b/src/tests/run_checkpoint_restart_test_with_prog_param.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# This version of the test runner doesn't attempt to run any parallel tests. +# We assume too, that if the library build has enabled MPI, that LD_LIBRARY_PATH is +# defined and points to the MPI libraries used by the linker (e.g. -L -lmpi) + +# Each argument should be a quoted string containing the executable and its parameters. + +# Usage: +# ./run_checkpoint_restart_test_with_prog_param.sh \ +# " [args...]" " [args...]" ... +# +# Example: +# ./run_checkpoint_restart_test_with_prog_param.sh \ +# "/path/to/kvtag_add_scale 1000 100" \ +# "/path/to/kvtag_get_verify_scale 1000 100" + +# Cori CI needs srun even for serial tests +run_cmd="" + +if [[ "$SUPERCOMPUTER" == "perlmutter" ]]; then + run_cmd="srun -n 1 --mem=25600 --cpu_bind=cores --overlap" +fi + +if [ $# -lt 1 ]; then echo "missing test argument" && exit -1; fi + +rm -rf pdc_data pdc_tmp + +restart=" " +ret=0 +for test_args in "$@" +do + # Split the quoted string into the executable and its parameters + read -ra cmd_parts <<< "$test_args" + test_exe="${cmd_parts[0]}" + test_params="${cmd_parts[@]:1}" + + if [ ! -x "$test_exe" ]; then + echo "test: $test_exe not found or not an executable" && exit -2 + fi + + # START the server (in the background) + echo "$run_cmd ./pdc_server $restart &" + $run_cmd "./pdc_server" $restart & + + # WAIT a bit... + sleep 1 + + echo "testing: $test_exe $test_params" + $run_cmd $test_exe $test_params + ret="$?" + + # Shutdown the server before next iteration + echo "$run_cmd ./close_server" + $run_cmd "./close_server" + + restart="restart" +done + +exit $ret \ No newline at end of file From d8171d9bfa83c92406b732966a85f62e810c4ed7 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 26 Jun 2026 11:07:35 -0400 Subject: [PATCH 18/30] Split kvtag scale tests into serial and MPI binaries from one source. Build kvtag_add_scale and kvtag_get_verify_scale as serial target (KVTAG_SCALE_SERIAL) and kvtag_add_scale_mpi / kvtag_get_verify_scale_mpi as MPI targets from the same .c files. Use a per-target macro so global ENABLE_MPI does not force MPI into the serial binaries. Initialize rank and timing for serial runs, and point MPI CTest and benchmark scripts at the _mpi executables. --- src/tests/CMakeLists.txt | 43 ++++++++++++++-- ...kpoint_restart_mpi_test_with_prog_param.sh | 4 +- src/tests/tags/kvtag_add_scale.c | 50 +++++++++++++++---- src/tests/tags/kvtag_get_verify_scale.c | 46 +++++++++++++---- 4 files changed, 118 insertions(+), 25 deletions(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 0a1743006..90c127b05 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,8 +102,6 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall - tags/kvtag_add_scale - tags/kvtag_get_verify_scale ) # TODO: Check if import_vpic.c is needed. If yes, we have to add the following : @@ -132,6 +130,45 @@ foreach(program ${PROGRAMS}) endif() endforeach() +# One source file, two targets: serial (KVTAG_SCALE_SERIAL) and MPI (_mpi). +add_executable(kvtag_add_scale tags/kvtag_add_scale.c) +target_compile_definitions(kvtag_add_scale PRIVATE KVTAG_SCALE_SERIAL=1) +target_include_directories(kvtag_add_scale PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/tests/include +) +target_link_libraries(kvtag_add_scale pdc pdc_commons ${TEST_EXT_LIB}) + +add_executable(kvtag_get_verify_scale tags/kvtag_get_verify_scale.c) +target_compile_definitions(kvtag_get_verify_scale PRIVATE KVTAG_SCALE_SERIAL=1) +target_include_directories(kvtag_get_verify_scale PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/tests/include +) +target_link_libraries(kvtag_get_verify_scale pdc pdc_commons ${TEST_EXT_LIB}) + +if(BUILD_MPI_TESTING) + add_executable(kvtag_add_scale_mpi tags/kvtag_add_scale.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(kvtag_add_scale_mpi PRIVATE ${SUPPRESSED_LIST}) + endif() + target_include_directories(kvtag_add_scale_mpi PRIVATE + ${TEST_EXT_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/src/tests/include + ) + target_link_libraries(kvtag_add_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) + + add_executable(kvtag_get_verify_scale_mpi tags/kvtag_get_verify_scale.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(kvtag_get_verify_scale_mpi PRIVATE ${SUPPRESSED_LIST}) + endif() + target_include_directories(kvtag_get_verify_scale_mpi PRIVATE + ${TEST_EXT_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/src/tests/include + ) + target_link_libraries(kvtag_get_verify_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) +endif() + if(BUILD_MPI_TESTING) set(MPI_PROGRAMS tags/kvtag_range_query_scale @@ -467,7 +504,7 @@ if(BUILD_MPI_TESTING) add_test(NAME obj_get_data_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./obj_get_data ${MPI_RUN_CMD} 4 6 ) add_test(NAME vpicio_bdcats_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_mpi_test.sh ${MPI_RUN_CMD} 4 6 ./vpicio ./bdcats) add_test(NAME vpicio_mpi_small_cache WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./vpicio ${MPI_RUN_CMD} 1 16) - add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") + add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale_mpi 1000 100" "./kvtag_get_verify_scale_mpi 1000 100") set_tests_properties(read_obj_shared_int PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(read_obj_shared_float PROPERTIES LABELS "parallel;parallel_obj" ) diff --git a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh index f4da849e3..db87b7f83 100755 --- a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh +++ b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh @@ -9,8 +9,8 @@ # # Example: # ./run_checkpoint_restart_mpi_test_with_prog_param.sh srun 2 4 \ -# "/path/to/kvtag_add_scale 1000 100" \ -# "/path/to/kvtag_get_verify_scale 1000 100" +# "/path/to/kvtag_add_scale_mpi 1000 100" \ +# "/path/to/kvtag_get_verify_scale_mpi 1000 100" extra_cmd="" diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 4e00097f6..33ecd5592 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -30,6 +30,23 @@ #include "pdc.h" #include "pdc_client_connect.h" +#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) +#define KVTAG_SCALE_USE_MPI 1 +#endif + +#ifdef KVTAG_SCALE_SERIAL +#include + +static double +kvtag_scale_now(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} +#endif + int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -78,10 +95,13 @@ main(int argc, char *argv[]) pdc_kvtag_t kvtag; int ret_value = SUCCEED; -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#else + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -127,9 +147,11 @@ main(int argc, char *argv[]) // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < my_obj; i++) { @@ -138,8 +160,8 @@ main(int argc, char *argv[]) if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to create object"); - if (i > 0 && i % obj_1percent == 0) { -#ifdef ENABLE_MPI + if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -153,9 +175,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -168,17 +192,19 @@ main(int argc, char *argv[]) kvtag.type = PDC_INT; kvtag.size = sizeof(int); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); - if (i % tag_1percent == 0) { -#ifdef ENABLE_MPI + if (tag_1percent > 0 && i % tag_1percent == 0) { +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -192,9 +218,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, @@ -208,9 +236,9 @@ main(int argc, char *argv[]) } done: -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Finalize(); #endif return ret_value; -} \ No newline at end of file +} diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index ab98d093d..d75dfcd91 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -30,6 +30,23 @@ #include "pdc.h" #include "pdc_client_connect.h" +#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) +#define KVTAG_SCALE_USE_MPI 1 +#endif + +#ifdef KVTAG_SCALE_SERIAL +#include + +static double +kvtag_scale_now(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} +#endif + int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -86,10 +103,13 @@ main(int argc, char *argv[]) int all_verified_success = 0; int all_verified_fail = 0; -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#else + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -124,9 +144,11 @@ main(int argc, char *argv[]) // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif // open already created PDC objects and query tags @@ -138,7 +160,7 @@ main(int argc, char *argv[]) // progress reporting for object opening if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -152,9 +174,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -166,9 +190,11 @@ main(int argc, char *argv[]) values = (void **)calloc(n_tag, sizeof(void *)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, @@ -192,9 +218,11 @@ main(int argc, char *argv[]) free(values[i]); } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, @@ -204,7 +232,7 @@ main(int argc, char *argv[]) free(obj_ids); // aggregate verification statistics across all ranks -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else @@ -237,9 +265,9 @@ main(int argc, char *argv[]) } done: -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Finalize(); #endif return ret_value; -} \ No newline at end of file +} From 374276585bd442d9b9e5cd42f285132cf0c274b3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 26 Jun 2026 15:08:08 +0000 Subject: [PATCH 19/30] Committing clang-format changes --- src/tests/tags/kvtag_add_scale.c | 8 ++++---- src/tests/tags/kvtag_get_verify_scale.c | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 33ecd5592..1da108709 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -100,8 +100,8 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #else - proc_num = 1; - my_rank = 0; + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -151,7 +151,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #else - stime = kvtag_scale_now(); + stime = kvtag_scale_now(); #endif for (i = 0; i < my_obj; i++) { @@ -196,7 +196,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #else - stime = kvtag_scale_now(); + stime = kvtag_scale_now(); #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index d75dfcd91..5d03faba7 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -108,8 +108,8 @@ main(int argc, char *argv[]) MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); #else - proc_num = 1; - my_rank = 0; + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -148,7 +148,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #else - stime = kvtag_scale_now(); + stime = kvtag_scale_now(); #endif // open already created PDC objects and query tags @@ -178,7 +178,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; #else - total_time = kvtag_scale_now() - stime; + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -194,7 +194,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); #else - stime = kvtag_scale_now(); + stime = kvtag_scale_now(); #endif for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, @@ -222,7 +222,7 @@ main(int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; #else - total_time = kvtag_scale_now() - stime; + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, From a772b68f7ad2de03787b15922c912fd44628cee6 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 3 Feb 2026 16:40:35 -0800 Subject: [PATCH 20/30] Merge pull request #302 from hpc-io/develop Sync develop to stable --- src/tests/CMakeLists.txt | 43 +++++++++++- src/tests/benchmark_strong_scaling.sh | 67 +++++++++++++++++++ src/tests/benchmark_weak_scaling.sh | 65 ++++++++++++++++++ ...kpoint_restart_mpi_test_with_prog_param.sh | 4 +- src/tests/tags/kvtag_add_scale.c | 50 +++++++++++--- src/tests/tags/kvtag_get_verify_scale.c | 46 ++++++++++--- 6 files changed, 250 insertions(+), 25 deletions(-) create mode 100755 src/tests/benchmark_strong_scaling.sh create mode 100755 src/tests/benchmark_weak_scaling.sh diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 0a1743006..90c127b05 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,8 +102,6 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall - tags/kvtag_add_scale - tags/kvtag_get_verify_scale ) # TODO: Check if import_vpic.c is needed. If yes, we have to add the following : @@ -132,6 +130,45 @@ foreach(program ${PROGRAMS}) endif() endforeach() +# One source file, two targets: serial (KVTAG_SCALE_SERIAL) and MPI (_mpi). +add_executable(kvtag_add_scale tags/kvtag_add_scale.c) +target_compile_definitions(kvtag_add_scale PRIVATE KVTAG_SCALE_SERIAL=1) +target_include_directories(kvtag_add_scale PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/tests/include +) +target_link_libraries(kvtag_add_scale pdc pdc_commons ${TEST_EXT_LIB}) + +add_executable(kvtag_get_verify_scale tags/kvtag_get_verify_scale.c) +target_compile_definitions(kvtag_get_verify_scale PRIVATE KVTAG_SCALE_SERIAL=1) +target_include_directories(kvtag_get_verify_scale PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/tests/include +) +target_link_libraries(kvtag_get_verify_scale pdc pdc_commons ${TEST_EXT_LIB}) + +if(BUILD_MPI_TESTING) + add_executable(kvtag_add_scale_mpi tags/kvtag_add_scale.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(kvtag_add_scale_mpi PRIVATE ${SUPPRESSED_LIST}) + endif() + target_include_directories(kvtag_add_scale_mpi PRIVATE + ${TEST_EXT_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/src/tests/include + ) + target_link_libraries(kvtag_add_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) + + add_executable(kvtag_get_verify_scale_mpi tags/kvtag_get_verify_scale.c) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(kvtag_get_verify_scale_mpi PRIVATE ${SUPPRESSED_LIST}) + endif() + target_include_directories(kvtag_get_verify_scale_mpi PRIVATE + ${TEST_EXT_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/src/tests/include + ) + target_link_libraries(kvtag_get_verify_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) +endif() + if(BUILD_MPI_TESTING) set(MPI_PROGRAMS tags/kvtag_range_query_scale @@ -467,7 +504,7 @@ if(BUILD_MPI_TESTING) add_test(NAME obj_get_data_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./obj_get_data ${MPI_RUN_CMD} 4 6 ) add_test(NAME vpicio_bdcats_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_mpi_test.sh ${MPI_RUN_CMD} 4 6 ./vpicio ./bdcats) add_test(NAME vpicio_mpi_small_cache WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./vpicio ${MPI_RUN_CMD} 1 16) - add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") + add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale_mpi 1000 100" "./kvtag_get_verify_scale_mpi 1000 100") set_tests_properties(read_obj_shared_int PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(read_obj_shared_float PROPERTIES LABELS "parallel;parallel_obj" ) diff --git a/src/tests/benchmark_strong_scaling.sh b/src/tests/benchmark_strong_scaling.sh new file mode 100755 index 000000000..74b1eef11 --- /dev/null +++ b/src/tests/benchmark_strong_scaling.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#export SUPERCOMPUTER=perlmutter +#module load PrgEnv-gnu +#module load libfabric +# +## note: might need for multi-node testing +##export FI_PROVIDER=tcp\;ofi_rxm +# +#export WORK_SPACE=/global/homes/r/raqib/pdc_home +#source $WORK_SPACE/pdc_env.sh +# +#export PDC_TMPDIR=$SCRATCH/pdc_tmp + +## Strong scaling (fix the total amount of work and then change the number of workers) +#Fix work to 100K/1000 and then changing the number of parallel servers: + #-n 2 -c 2 + #-n 4 -c 2 + #-n 8 -c 2 + #-n 16 -c 2 + +# parse log using: + # less 100K-100__2-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 100K-100__4-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 100K-100__8-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 100K-100__16-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + +for i in {1..3}; do + echo "===== Run $i ===== 100K-100__2-2" + echo "===== Run $i =====" >> 100K-100__2-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 2 1 \ + "./kvtag_add_scale_mpi 100000 100" \ + "./kvtag_get_verify_scale_mpi 100000 100" \ + >> 100K-100__2-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 100K-100__4-2" + echo "===== Run $i =====" >> 100K-100__4-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 4 1 \ + "./kvtag_add_scale_mpi 100000 100" \ + "./kvtag_get_verify_scale_mpi 100000 100" \ + >> 100K-100__4-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 100K-100__8-2" + echo "===== Run $i =====" >> 100K-100__8-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 8 1 \ + "./kvtag_add_scale_mpi 100000 100" \ + "./kvtag_get_verify_scale_mpi 100000 100" \ + >> 100K-100__8-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 100K-100__16-2" + echo "===== Run $i =====" >> 100K-100__16-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 16 1 \ + "./kvtag_add_scale_mpi 100000 100" \ + "./kvtag_get_verify_scale_mpi 100000 100" \ + >> 100K-100__16-2.out 2>&1 +done + diff --git a/src/tests/benchmark_weak_scaling.sh b/src/tests/benchmark_weak_scaling.sh new file mode 100755 index 000000000..b48bac197 --- /dev/null +++ b/src/tests/benchmark_weak_scaling.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +#module load PrgEnv-gnu +#module load libfabric +#export SUPERCOMPUTER=perlmutter +# +## note: might need for multi-node testing +##export FI_PROVIDER=tcp\;ofi_rxm +# +#export WORK_SPACE=/global/homes/r/raqib/pdc_home +#source $WORK_SPACE/pdc_env.sh +# +#export PDC_TMPDIR=$SCRATCH/pdc_tmp + +## Weak scaling (fix the amount of workload per-worker) + #10K/100 -n 2 -c 2 will implies in respect to checkpointing (10k + 10k*100) = 1010000 + #20K/100 -n 4 -c 2 will implies in respect to checkpointing (20k + 20k*100) = 2020000 + #40K/100 -n 8 -c 2 will implies in respect to checkpointing (40k + 40k*100) = 4040000 + #80K/100 -n 16 -c 2 will implies in respect to checkpointing (80k + 80k*100) = 8080000 + +# parse log using: + # less 10K-100__2-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 20K-100__4-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 40K-100__8-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + # less 80K-100__16-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" + +for i in {1..3}; do + echo "===== Run $i ===== 10K-100__2-2" + echo "===== Run $i =====" >> 10K-100__2-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 2 1 \ + "./kvtag_add_scale_mpi 10000 100" \ + "./kvtag_get_verify_scale_mpi 10000 100" \ + >> 10K-100__2-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 20K-100__4-2" + echo "===== Run $i =====" >> 20K-100__4-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 4 1 \ + "./kvtag_add_scale_mpi 20000 100" \ + "./kvtag_get_verify_scale_mpi 20000 100" \ + >> 20K-100__4-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 40K-100__8-2" + echo "===== Run $i =====" >> 40K-100__8-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 8 1 \ + "./kvtag_add_scale_mpi 40000 100" \ + "./kvtag_get_verify_scale_mpi 40000 100" \ + >> 40K-100__8-2.out 2>&1 +done + +for i in {1..3}; do + echo "===== Run $i ===== 80K-100__16-2" + echo "===== Run $i =====" >> 80K-100__16-2.out + ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ + srun 16 1 \ + "./kvtag_add_scale_mpi 80000 100" \ + "./kvtag_get_verify_scale_mpi 80000 100" \ + >> 80K-100__16-2.out 2>&1 +done diff --git a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh index f4da849e3..db87b7f83 100755 --- a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh +++ b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh @@ -9,8 +9,8 @@ # # Example: # ./run_checkpoint_restart_mpi_test_with_prog_param.sh srun 2 4 \ -# "/path/to/kvtag_add_scale 1000 100" \ -# "/path/to/kvtag_get_verify_scale 1000 100" +# "/path/to/kvtag_add_scale_mpi 1000 100" \ +# "/path/to/kvtag_get_verify_scale_mpi 1000 100" extra_cmd="" diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 4e00097f6..33ecd5592 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -30,6 +30,23 @@ #include "pdc.h" #include "pdc_client_connect.h" +#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) +#define KVTAG_SCALE_USE_MPI 1 +#endif + +#ifdef KVTAG_SCALE_SERIAL +#include + +static double +kvtag_scale_now(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} +#endif + int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -78,10 +95,13 @@ main(int argc, char *argv[]) pdc_kvtag_t kvtag; int ret_value = SUCCEED; -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#else + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -127,9 +147,11 @@ main(int argc, char *argv[]) // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < my_obj; i++) { @@ -138,8 +160,8 @@ main(int argc, char *argv[]) if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to create object"); - if (i > 0 && i % obj_1percent == 0) { -#ifdef ENABLE_MPI + if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -153,9 +175,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -168,17 +192,19 @@ main(int argc, char *argv[]) kvtag.type = PDC_INT; kvtag.size = sizeof(int); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); - if (i % tag_1percent == 0) { -#ifdef ENABLE_MPI + if (tag_1percent > 0 && i % tag_1percent == 0) { +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -192,9 +218,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, @@ -208,9 +236,9 @@ main(int argc, char *argv[]) } done: -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Finalize(); #endif return ret_value; -} \ No newline at end of file +} diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index ab98d093d..d75dfcd91 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -30,6 +30,23 @@ #include "pdc.h" #include "pdc_client_connect.h" +#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) +#define KVTAG_SCALE_USE_MPI 1 +#endif + +#ifdef KVTAG_SCALE_SERIAL +#include + +static double +kvtag_scale_now(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; +} +#endif + int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -86,10 +103,13 @@ main(int argc, char *argv[]) int all_verified_success = 0; int all_verified_fail = 0; -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); +#else + proc_num = 1; + my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -124,9 +144,11 @@ main(int argc, char *argv[]) // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif // open already created PDC objects and query tags @@ -138,7 +160,7 @@ main(int argc, char *argv[]) // progress reporting for object opening if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -152,9 +174,11 @@ main(int argc, char *argv[]) } } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -166,9 +190,11 @@ main(int argc, char *argv[]) values = (void **)calloc(n_tag, sizeof(void *)); -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); +#else + stime = kvtag_scale_now(); #endif for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, @@ -192,9 +218,11 @@ main(int argc, char *argv[]) free(values[i]); } -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; +#else + total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, @@ -204,7 +232,7 @@ main(int argc, char *argv[]) free(obj_ids); // aggregate verification statistics across all ranks -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else @@ -237,9 +265,9 @@ main(int argc, char *argv[]) } done: -#ifdef ENABLE_MPI +#ifdef KVTAG_SCALE_USE_MPI MPI_Finalize(); #endif return ret_value; -} \ No newline at end of file +} From be5e57f423d1b45317e3669ed5bd322b5dfb2d36 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Sat, 27 Jun 2026 12:35:27 -0400 Subject: [PATCH 21/30] Revert "Merge pull request #302 from hpc-io/develop" This reverts commit a772b68f7ad2de03787b15922c912fd44628cee6. --- src/tests/CMakeLists.txt | 43 +----------- src/tests/benchmark_strong_scaling.sh | 67 ------------------- src/tests/benchmark_weak_scaling.sh | 65 ------------------ ...kpoint_restart_mpi_test_with_prog_param.sh | 4 +- src/tests/tags/kvtag_add_scale.c | 50 +++----------- src/tests/tags/kvtag_get_verify_scale.c | 46 +++---------- 6 files changed, 25 insertions(+), 250 deletions(-) delete mode 100755 src/tests/benchmark_strong_scaling.sh delete mode 100755 src/tests/benchmark_weak_scaling.sh diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 90c127b05..0a1743006 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,6 +102,8 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall + tags/kvtag_add_scale + tags/kvtag_get_verify_scale ) # TODO: Check if import_vpic.c is needed. If yes, we have to add the following : @@ -130,45 +132,6 @@ foreach(program ${PROGRAMS}) endif() endforeach() -# One source file, two targets: serial (KVTAG_SCALE_SERIAL) and MPI (_mpi). -add_executable(kvtag_add_scale tags/kvtag_add_scale.c) -target_compile_definitions(kvtag_add_scale PRIVATE KVTAG_SCALE_SERIAL=1) -target_include_directories(kvtag_add_scale PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/tests/include -) -target_link_libraries(kvtag_add_scale pdc pdc_commons ${TEST_EXT_LIB}) - -add_executable(kvtag_get_verify_scale tags/kvtag_get_verify_scale.c) -target_compile_definitions(kvtag_get_verify_scale PRIVATE KVTAG_SCALE_SERIAL=1) -target_include_directories(kvtag_get_verify_scale PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/tests/include -) -target_link_libraries(kvtag_get_verify_scale pdc pdc_commons ${TEST_EXT_LIB}) - -if(BUILD_MPI_TESTING) - add_executable(kvtag_add_scale_mpi tags/kvtag_add_scale.c) - if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(kvtag_add_scale_mpi PRIVATE ${SUPPRESSED_LIST}) - endif() - target_include_directories(kvtag_add_scale_mpi PRIVATE - ${TEST_EXT_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/src/tests/include - ) - target_link_libraries(kvtag_add_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) - - add_executable(kvtag_get_verify_scale_mpi tags/kvtag_get_verify_scale.c) - if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(kvtag_get_verify_scale_mpi PRIVATE ${SUPPRESSED_LIST}) - endif() - target_include_directories(kvtag_get_verify_scale_mpi PRIVATE - ${TEST_EXT_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/src/tests/include - ) - target_link_libraries(kvtag_get_verify_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) -endif() - if(BUILD_MPI_TESTING) set(MPI_PROGRAMS tags/kvtag_range_query_scale @@ -504,7 +467,7 @@ if(BUILD_MPI_TESTING) add_test(NAME obj_get_data_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./obj_get_data ${MPI_RUN_CMD} 4 6 ) add_test(NAME vpicio_bdcats_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_mpi_test.sh ${MPI_RUN_CMD} 4 6 ./vpicio ./bdcats) add_test(NAME vpicio_mpi_small_cache WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./vpicio ${MPI_RUN_CMD} 1 16) - add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale_mpi 1000 100" "./kvtag_get_verify_scale_mpi 1000 100") + add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") set_tests_properties(read_obj_shared_int PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(read_obj_shared_float PROPERTIES LABELS "parallel;parallel_obj" ) diff --git a/src/tests/benchmark_strong_scaling.sh b/src/tests/benchmark_strong_scaling.sh deleted file mode 100755 index 74b1eef11..000000000 --- a/src/tests/benchmark_strong_scaling.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -#export SUPERCOMPUTER=perlmutter -#module load PrgEnv-gnu -#module load libfabric -# -## note: might need for multi-node testing -##export FI_PROVIDER=tcp\;ofi_rxm -# -#export WORK_SPACE=/global/homes/r/raqib/pdc_home -#source $WORK_SPACE/pdc_env.sh -# -#export PDC_TMPDIR=$SCRATCH/pdc_tmp - -## Strong scaling (fix the total amount of work and then change the number of workers) -#Fix work to 100K/1000 and then changing the number of parallel servers: - #-n 2 -c 2 - #-n 4 -c 2 - #-n 8 -c 2 - #-n 16 -c 2 - -# parse log using: - # less 100K-100__2-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 100K-100__4-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 100K-100__8-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 100K-100__16-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - -for i in {1..3}; do - echo "===== Run $i ===== 100K-100__2-2" - echo "===== Run $i =====" >> 100K-100__2-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 2 1 \ - "./kvtag_add_scale_mpi 100000 100" \ - "./kvtag_get_verify_scale_mpi 100000 100" \ - >> 100K-100__2-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 100K-100__4-2" - echo "===== Run $i =====" >> 100K-100__4-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 4 1 \ - "./kvtag_add_scale_mpi 100000 100" \ - "./kvtag_get_verify_scale_mpi 100000 100" \ - >> 100K-100__4-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 100K-100__8-2" - echo "===== Run $i =====" >> 100K-100__8-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 8 1 \ - "./kvtag_add_scale_mpi 100000 100" \ - "./kvtag_get_verify_scale_mpi 100000 100" \ - >> 100K-100__8-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 100K-100__16-2" - echo "===== Run $i =====" >> 100K-100__16-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 16 1 \ - "./kvtag_add_scale_mpi 100000 100" \ - "./kvtag_get_verify_scale_mpi 100000 100" \ - >> 100K-100__16-2.out 2>&1 -done - diff --git a/src/tests/benchmark_weak_scaling.sh b/src/tests/benchmark_weak_scaling.sh deleted file mode 100755 index b48bac197..000000000 --- a/src/tests/benchmark_weak_scaling.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -#module load PrgEnv-gnu -#module load libfabric -#export SUPERCOMPUTER=perlmutter -# -## note: might need for multi-node testing -##export FI_PROVIDER=tcp\;ofi_rxm -# -#export WORK_SPACE=/global/homes/r/raqib/pdc_home -#source $WORK_SPACE/pdc_env.sh -# -#export PDC_TMPDIR=$SCRATCH/pdc_tmp - -## Weak scaling (fix the amount of workload per-worker) - #10K/100 -n 2 -c 2 will implies in respect to checkpointing (10k + 10k*100) = 1010000 - #20K/100 -n 4 -c 2 will implies in respect to checkpointing (20k + 20k*100) = 2020000 - #40K/100 -n 8 -c 2 will implies in respect to checkpointing (40k + 40k*100) = 4040000 - #80K/100 -n 16 -c 2 will implies in respect to checkpointing (80k + 80k*100) = 8080000 - -# parse log using: - # less 10K-100__2-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 20K-100__4-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 40K-100__8-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - # less 80K-100__16-2.out | grep "Total checkpoint time\|total close time\|===== Run\|total restart time" - -for i in {1..3}; do - echo "===== Run $i ===== 10K-100__2-2" - echo "===== Run $i =====" >> 10K-100__2-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 2 1 \ - "./kvtag_add_scale_mpi 10000 100" \ - "./kvtag_get_verify_scale_mpi 10000 100" \ - >> 10K-100__2-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 20K-100__4-2" - echo "===== Run $i =====" >> 20K-100__4-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 4 1 \ - "./kvtag_add_scale_mpi 20000 100" \ - "./kvtag_get_verify_scale_mpi 20000 100" \ - >> 20K-100__4-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 40K-100__8-2" - echo "===== Run $i =====" >> 40K-100__8-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 8 1 \ - "./kvtag_add_scale_mpi 40000 100" \ - "./kvtag_get_verify_scale_mpi 40000 100" \ - >> 40K-100__8-2.out 2>&1 -done - -for i in {1..3}; do - echo "===== Run $i ===== 80K-100__16-2" - echo "===== Run $i =====" >> 80K-100__16-2.out - ./run_checkpoint_restart_mpi_test_with_prog_param.sh \ - srun 16 1 \ - "./kvtag_add_scale_mpi 80000 100" \ - "./kvtag_get_verify_scale_mpi 80000 100" \ - >> 80K-100__16-2.out 2>&1 -done diff --git a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh index db87b7f83..f4da849e3 100755 --- a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh +++ b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh @@ -9,8 +9,8 @@ # # Example: # ./run_checkpoint_restart_mpi_test_with_prog_param.sh srun 2 4 \ -# "/path/to/kvtag_add_scale_mpi 1000 100" \ -# "/path/to/kvtag_get_verify_scale_mpi 1000 100" +# "/path/to/kvtag_add_scale 1000 100" \ +# "/path/to/kvtag_get_verify_scale 1000 100" extra_cmd="" diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 33ecd5592..4e00097f6 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -30,23 +30,6 @@ #include "pdc.h" #include "pdc_client_connect.h" -#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) -#define KVTAG_SCALE_USE_MPI 1 -#endif - -#ifdef KVTAG_SCALE_SERIAL -#include - -static double -kvtag_scale_now(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} -#endif - int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -95,13 +78,10 @@ main(int argc, char *argv[]) pdc_kvtag_t kvtag; int ret_value = SUCCEED; -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); -#else - proc_num = 1; - my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -147,11 +127,9 @@ main(int argc, char *argv[]) // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < my_obj; i++) { @@ -160,8 +138,8 @@ main(int argc, char *argv[]) if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to create object"); - if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI + if (i > 0 && i % obj_1percent == 0) { +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -175,11 +153,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -192,19 +168,17 @@ main(int argc, char *argv[]) kvtag.type = PDC_INT; kvtag.size = sizeof(int); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); - if (tag_1percent > 0 && i % tag_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI + if (i % tag_1percent == 0) { +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -218,11 +192,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, @@ -236,9 +208,9 @@ main(int argc, char *argv[]) } done: -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Finalize(); #endif return ret_value; -} +} \ No newline at end of file diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index d75dfcd91..ab98d093d 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -30,23 +30,6 @@ #include "pdc.h" #include "pdc_client_connect.h" -#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) -#define KVTAG_SCALE_USE_MPI 1 -#endif - -#ifdef KVTAG_SCALE_SERIAL -#include - -static double -kvtag_scale_now(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} -#endif - int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -103,13 +86,10 @@ main(int argc, char *argv[]) int all_verified_success = 0; int all_verified_fail = 0; -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); -#else - proc_num = 1; - my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -144,11 +124,9 @@ main(int argc, char *argv[]) // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif // open already created PDC objects and query tags @@ -160,7 +138,7 @@ main(int argc, char *argv[]) // progress reporting for object opening if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -174,11 +152,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -190,11 +166,9 @@ main(int argc, char *argv[]) values = (void **)calloc(n_tag, sizeof(void *)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, @@ -218,11 +192,9 @@ main(int argc, char *argv[]) free(values[i]); } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, @@ -232,7 +204,7 @@ main(int argc, char *argv[]) free(obj_ids); // aggregate verification statistics across all ranks -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else @@ -265,9 +237,9 @@ main(int argc, char *argv[]) } done: -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Finalize(); #endif return ret_value; -} +} \ No newline at end of file From 01d9f3ad86cb9e2189d1e657b023dcfdde76f502 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Sat, 27 Jun 2026 12:45:58 -0400 Subject: [PATCH 22/30] Revert "Split kvtag scale tests into serial and MPI binaries from one source." This reverts commit d8171d9bfa83c92406b732966a85f62e810c4ed7. --- src/tests/CMakeLists.txt | 43 ++-------------- ...kpoint_restart_mpi_test_with_prog_param.sh | 4 +- src/tests/tags/kvtag_add_scale.c | 50 ++++--------------- src/tests/tags/kvtag_get_verify_scale.c | 46 ++++------------- 4 files changed, 25 insertions(+), 118 deletions(-) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 90c127b05..0a1743006 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,6 +102,8 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall + tags/kvtag_add_scale + tags/kvtag_get_verify_scale ) # TODO: Check if import_vpic.c is needed. If yes, we have to add the following : @@ -130,45 +132,6 @@ foreach(program ${PROGRAMS}) endif() endforeach() -# One source file, two targets: serial (KVTAG_SCALE_SERIAL) and MPI (_mpi). -add_executable(kvtag_add_scale tags/kvtag_add_scale.c) -target_compile_definitions(kvtag_add_scale PRIVATE KVTAG_SCALE_SERIAL=1) -target_include_directories(kvtag_add_scale PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/tests/include -) -target_link_libraries(kvtag_add_scale pdc pdc_commons ${TEST_EXT_LIB}) - -add_executable(kvtag_get_verify_scale tags/kvtag_get_verify_scale.c) -target_compile_definitions(kvtag_get_verify_scale PRIVATE KVTAG_SCALE_SERIAL=1) -target_include_directories(kvtag_get_verify_scale PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/tests/include -) -target_link_libraries(kvtag_get_verify_scale pdc pdc_commons ${TEST_EXT_LIB}) - -if(BUILD_MPI_TESTING) - add_executable(kvtag_add_scale_mpi tags/kvtag_add_scale.c) - if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(kvtag_add_scale_mpi PRIVATE ${SUPPRESSED_LIST}) - endif() - target_include_directories(kvtag_add_scale_mpi PRIVATE - ${TEST_EXT_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/src/tests/include - ) - target_link_libraries(kvtag_add_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) - - add_executable(kvtag_get_verify_scale_mpi tags/kvtag_get_verify_scale.c) - if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(kvtag_get_verify_scale_mpi PRIVATE ${SUPPRESSED_LIST}) - endif() - target_include_directories(kvtag_get_verify_scale_mpi PRIVATE - ${TEST_EXT_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/src/tests/include - ) - target_link_libraries(kvtag_get_verify_scale_mpi pdc pdc_commons ${TEST_EXT_LIB}) -endif() - if(BUILD_MPI_TESTING) set(MPI_PROGRAMS tags/kvtag_range_query_scale @@ -504,7 +467,7 @@ if(BUILD_MPI_TESTING) add_test(NAME obj_get_data_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./obj_get_data ${MPI_RUN_CMD} 4 6 ) add_test(NAME vpicio_bdcats_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_multiple_mpi_test.sh ${MPI_RUN_CMD} 4 6 ./vpicio ./bdcats) add_test(NAME vpicio_mpi_small_cache WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./vpicio ${MPI_RUN_CMD} 1 16) - add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale_mpi 1000 100" "./kvtag_get_verify_scale_mpi 1000 100") + add_test(NAME kvtag_checkpoint_restart_mpi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_checkpoint_restart_mpi_test_with_prog_param.sh ${MPI_RUN_CMD} 2 4 "./kvtag_add_scale 1000 100" "./kvtag_get_verify_scale 1000 100") set_tests_properties(read_obj_shared_int PROPERTIES LABELS "parallel;parallel_obj" ) set_tests_properties(read_obj_shared_float PROPERTIES LABELS "parallel;parallel_obj" ) diff --git a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh index db87b7f83..f4da849e3 100755 --- a/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh +++ b/src/tests/run_checkpoint_restart_mpi_test_with_prog_param.sh @@ -9,8 +9,8 @@ # # Example: # ./run_checkpoint_restart_mpi_test_with_prog_param.sh srun 2 4 \ -# "/path/to/kvtag_add_scale_mpi 1000 100" \ -# "/path/to/kvtag_get_verify_scale_mpi 1000 100" +# "/path/to/kvtag_add_scale 1000 100" \ +# "/path/to/kvtag_get_verify_scale 1000 100" extra_cmd="" diff --git a/src/tests/tags/kvtag_add_scale.c b/src/tests/tags/kvtag_add_scale.c index 1da108709..4e00097f6 100644 --- a/src/tests/tags/kvtag_add_scale.c +++ b/src/tests/tags/kvtag_add_scale.c @@ -30,23 +30,6 @@ #include "pdc.h" #include "pdc_client_connect.h" -#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) -#define KVTAG_SCALE_USE_MPI 1 -#endif - -#ifdef KVTAG_SCALE_SERIAL -#include - -static double -kvtag_scale_now(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} -#endif - int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -95,13 +78,10 @@ main(int argc, char *argv[]) pdc_kvtag_t kvtag; int ret_value = SUCCEED; -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); -#else - proc_num = 1; - my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -147,11 +127,9 @@ main(int argc, char *argv[]) // create a number of objects, add at least one tag to that object obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < my_obj; i++) { @@ -160,8 +138,8 @@ main(int argc, char *argv[]) if (obj_ids[i] <= 0) PGOTO_ERROR(FAIL, "Failed to create object"); - if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI + if (i > 0 && i % obj_1percent == 0) { +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -175,11 +153,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -192,19 +168,17 @@ main(int argc, char *argv[]) kvtag.type = PDC_INT; kvtag.size = sizeof(int); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < my_add_tag; i++) { v = i + my_add_tag_s; if (PDCobj_put_tag(obj_ids[i], kvtag.name, kvtag.value, kvtag.type, kvtag.size) < 0) PGOTO_ERROR(FAIL, "Failed to add a kvtag to o%d", i + my_obj_s); - if (tag_1percent > 0 && i % tag_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI + if (i % tag_1percent == 0) { +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -218,11 +192,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to add tags to %11d objects: %7.2f , throughput %10.2f \n", n_add_tag, @@ -236,9 +208,9 @@ main(int argc, char *argv[]) } done: -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Finalize(); #endif return ret_value; -} +} \ No newline at end of file diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index 5d03faba7..ab98d093d 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -30,23 +30,6 @@ #include "pdc.h" #include "pdc_client_connect.h" -#if defined(ENABLE_MPI) && !defined(KVTAG_SCALE_SERIAL) -#define KVTAG_SCALE_USE_MPI 1 -#endif - -#ifdef KVTAG_SCALE_SERIAL -#include - -static double -kvtag_scale_now(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -} -#endif - int assign_work_to_rank(int rank, int size, int nwork, int *my_count, int *my_start) { @@ -103,13 +86,10 @@ main(int argc, char *argv[]) int all_verified_success = 0; int all_verified_fail = 0; -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &proc_num); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); -#else - proc_num = 1; - my_rank = 0; #endif if (argc < 3) { if (my_rank == 0) @@ -144,11 +124,9 @@ main(int argc, char *argv[]) // Open existing objects obj_ids = (pdcid_t *)calloc(my_obj, sizeof(pdcid_t)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif // open already created PDC objects and query tags @@ -160,7 +138,7 @@ main(int argc, char *argv[]) // progress reporting for object opening if (i > 0 && obj_1percent > 0 && i % obj_1percent == 0) { -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); percent_time = MPI_Wtime() - stime; if (my_rank == 0) { @@ -174,11 +152,9 @@ main(int argc, char *argv[]) } } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) @@ -190,11 +166,9 @@ main(int argc, char *argv[]) values = (void **)calloc(n_tag, sizeof(void *)); -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); stime = MPI_Wtime(); -#else - stime = kvtag_scale_now(); #endif for (i = 0; i < n_tag; i++) { if (PDCobj_get_tag(obj_ids[i], kvtag.name, (void *)&values[i], (void *)&value_type, @@ -218,11 +192,9 @@ main(int argc, char *argv[]) free(values[i]); } -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); total_time = MPI_Wtime() - stime; -#else - total_time = kvtag_scale_now() - stime; #endif if (my_rank == 0) LOG_INFO("Total time to retrieve %11d tag from %11d objects: %7.2f , throughput %10.2f \n", n_query, @@ -232,7 +204,7 @@ main(int argc, char *argv[]) free(obj_ids); // aggregate verification statistics across all ranks -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Reduce(&verified_success, &all_verified_success, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&verified_fail, &all_verified_fail, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); #else @@ -265,9 +237,9 @@ main(int argc, char *argv[]) } done: -#ifdef KVTAG_SCALE_USE_MPI +#ifdef ENABLE_MPI MPI_Finalize(); #endif return ret_value; -} +} \ No newline at end of file From dc699f2019ecc8b95c23f0c48f475a89a0cdd612 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Thu, 2 Jul 2026 15:34:05 -0400 Subject: [PATCH 23/30] Harden BULKI checkpoint write and restart against malformed input. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create checkpoint files with mode 0600 and validate untrusted BULKI restart fields (metadata counts, kvtag key/value sizes, region counts, and histogram nbin) before allocation, matching develop-branch's binary checkpoint safeguards. Applied changes 1. Checkpoint (PDC_Server_checkpoint) a. Added shared bounds constants (metadata count, kvtag key/size, region count, histogram nbin). b. Checkpoint files created with open(..., O_CREAT | O_TRUNC, 0600) + fdopen, matching develop’s permission model. 2. Restart (PDC_Server_restart) a. Metadata count: reject if count <= 0 or > 10,000,000. b. KV tag key: null checks; key length capped at 65536 (uses key_ent->count when set). c. KV tag value: size must be > 0 and ≤ 16 MiB; value_ent->count must match. d. Regions (metadata + dataserver): reject if n_region < 0 or > 1,000,000. e. Histogram: nbin must be > 0 and ≤ 65536 before allocation; uses validated nbin for malloc sizes. --- src/server/pdc_server.c | 71 +++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 905cd9694..58225434b 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -90,6 +90,13 @@ sqlite3 *sqlite3_db_g; // Checkpoint every INTERVAL_SEC second and at least OP_INTERVAL operations #define PDC_CHECKPOINT_MIN_INTERVAL_SEC 1800 +// Bounds for untrusted checkpoint data +#define PDC_CHECKPOINT_MAX_METADATA_COUNT 10000000 +#define PDC_CHECKPOINT_MAX_KVTAG_KEY_LEN 65536 +#define PDC_CHECKPOINT_MAX_KVTAG_SIZE (1u << 24) +#define PDC_CHECKPOINT_MAX_REGION_COUNT 1000000 +#define PDC_CHECKPOINT_MAX_HIST_NBIN 65536 + // Global debug variable to control debug printfs int is_debug_g = 0; int pdc_client_num_g = 0; @@ -1479,12 +1486,16 @@ PDC_Server_checkpoint() // ========== Serialize and Write ========== if (use_tmpfs) - file = fopen(checkpoint_file_local, "wb"); + fd = open(checkpoint_file_local, O_RDWR | O_CREAT | O_TRUNC, 0600); else - file = fopen(checkpoint_file, "wb"); - - if (file == NULL) + fd = open(checkpoint_file, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (fd < 0) + PGOTO_ERROR(FAIL, "Failed to open checkpoint file"); + file = fdopen(fd, "wb"); + if (file == NULL) { + close(fd); PGOTO_ERROR(FAIL, "Checkpoint file open error"); + } // note: BULKI_Entity_serialize_to_file() still do in-memory serialization // note: need to implement a streaming file writing without in-memory serialization @@ -1667,7 +1678,11 @@ PDC_Server_restart(char *filename) // extract n_obj BULKI_Entity *n_obj_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("n_obj", PDC_STRING)); int count; + if (n_obj_ent == NULL) + PGOTO_ERROR(FAIL, "Missing n_obj in checkpoint metadata entry"); memcpy(&count, n_obj_ent->data, sizeof(int)); + if (count <= 0 || count > PDC_CHECKPOINT_MAX_METADATA_COUNT) + PGOTO_ERROR(FAIL, "Suspicious count value %d from checkpoint", count); // extract hash key BULKI_Entity *hash_key_ent = @@ -1730,14 +1745,27 @@ PDC_Server_restart(char *filename) // extract key BULKI_Entity *key_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("key", PDC_STRING)); - int key_len = strlen((char *)key_ent->data) + 1; + + if (key_ent == NULL || key_ent->data == NULL) + PGOTO_ERROR(FAIL, "Invalid kvtag key in checkpoint"); + + int key_len = (int)strlen((char *)key_ent->data) + 1; + if (key_len <= 0 || key_len > PDC_CHECKPOINT_MAX_KVTAG_KEY_LEN) + PGOTO_ERROR(FAIL, "Invalid key_len %d in checkpoint", key_len); + kvtag_list->kvtag->name = PDC_malloc(key_len); memcpy(kvtag_list->kvtag->name, key_ent->data, key_len); // extract size BULKI_Entity *size_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("size", PDC_STRING)); - memcpy(&kvtag_list->kvtag->size, size_ent->data, sizeof(uint32_t)); + if (size_ent == NULL) + PGOTO_ERROR(FAIL, "Missing kvtag size in checkpoint"); + uint32_t kv_size; + memcpy(&kv_size, size_ent->data, sizeof(uint32_t)); + if (kv_size == 0 || kv_size > PDC_CHECKPOINT_MAX_KVTAG_SIZE) + PGOTO_ERROR(FAIL, "Invalid kvtag size in checkpoint"); + kvtag_list->kvtag->size = kv_size; // extract type BULKI_Entity *type_ent = @@ -1747,8 +1775,10 @@ PDC_Server_restart(char *filename) // extract value BULKI_Entity *value_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("value", PDC_STRING)); - kvtag_list->kvtag->value = PDC_malloc(kvtag_list->kvtag->size); - memcpy(kvtag_list->kvtag->value, value_ent->data, kvtag_list->kvtag->size); + if (value_ent == NULL || value_ent->count != kv_size) + PGOTO_ERROR(FAIL, "Invalid kvtag value size in checkpoint"); + kvtag_list->kvtag->value = PDC_malloc((size_t)kv_size); + memcpy(kvtag_list->kvtag->value, value_ent->data, (size_t)kv_size); DL_APPEND((metadata + i)->kvtag_list_head, kvtag_list); } @@ -1761,6 +1791,8 @@ PDC_Server_restart(char *filename) int n_region = 0; if (regions_array != NULL && regions_array->pdc_type == PDC_BULKI) { n_region = regions_array->count; + if (n_region < 0 || n_region > PDC_CHECKPOINT_MAX_REGION_COUNT) + PGOTO_ERROR(FAIL, "Suspicious n_region value %d from checkpoint", n_region); BULKI_Entity_Iterator *region_iter = Bent_iterator_init(regions_array, NULL, PDC_BULKI); @@ -1796,25 +1828,27 @@ PDC_Server_restart(char *filename) BULKI_Entity *nbin_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("nbin", PDC_STRING)); - memcpy(®ion_list->region_hist->nbin, nbin_ent->data, sizeof(int)); - - if (region_list->region_hist->nbin == 0) { - LOG_ERROR("Checkpoint file histogram size is 0\n"); - } + int nbin; + if (nbin_ent == NULL) + PGOTO_ERROR(FAIL, "Missing histogram nbin in checkpoint"); + memcpy(&nbin, nbin_ent->data, sizeof(int)); + if (nbin <= 0 || nbin > PDC_CHECKPOINT_MAX_HIST_NBIN) + PGOTO_ERROR(FAIL, "Invalid histogram nbin %d in checkpoint", nbin); + region_list->region_hist->nbin = nbin; BULKI_Entity *range_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("range", PDC_STRING)); region_list->region_hist->range = (double *)PDC_malloc( - sizeof(double) * region_list->region_hist->nbin * 2); + sizeof(double) * (size_t)nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, - sizeof(double) * region_list->region_hist->nbin * 2); + sizeof(double) * (size_t)nbin * 2); BULKI_Entity *bin_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); region_list->region_hist->bin = (uint64_t *)PDC_malloc( - sizeof(uint64_t) * region_list->region_hist->nbin); + sizeof(uint64_t) * (size_t)nbin); memcpy(region_list->region_hist->bin, bin_ent->data, - sizeof(uint64_t) * region_list->region_hist->nbin); + sizeof(uint64_t) * (size_t)nbin); BULKI_Entity *incr_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING)); @@ -1910,6 +1944,9 @@ PDC_Server_restart(char *filename) BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("regions", PDC_STRING)); if (ds_regions_array != NULL && ds_regions_array->pdc_type == PDC_BULKI) { + int ds_n_region = ds_regions_array->count; + if (ds_n_region < 0 || ds_n_region > PDC_CHECKPOINT_MAX_REGION_COUNT) + PGOTO_ERROR(FAIL, "Suspicious dataserver n_region value %d from checkpoint", ds_n_region); BULKI_Entity_Iterator *ds_region_iter = Bent_iterator_init(ds_regions_array, NULL, PDC_BULKI); while (Bent_iterator_has_next_BULKI(ds_region_iter)) { From a43c8e31705b4e47b6e36ef5582fb0f7af6b0d7f Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 2 Jul 2026 19:34:40 +0000 Subject: [PATCH 24/30] Committing clang-format changes --- src/server/pdc_server.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 58225434b..3827a519e 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -91,11 +91,11 @@ sqlite3 *sqlite3_db_g; #define PDC_CHECKPOINT_MIN_INTERVAL_SEC 1800 // Bounds for untrusted checkpoint data -#define PDC_CHECKPOINT_MAX_METADATA_COUNT 10000000 -#define PDC_CHECKPOINT_MAX_KVTAG_KEY_LEN 65536 -#define PDC_CHECKPOINT_MAX_KVTAG_SIZE (1u << 24) -#define PDC_CHECKPOINT_MAX_REGION_COUNT 1000000 -#define PDC_CHECKPOINT_MAX_HIST_NBIN 65536 +#define PDC_CHECKPOINT_MAX_METADATA_COUNT 10000000 +#define PDC_CHECKPOINT_MAX_KVTAG_KEY_LEN 65536 +#define PDC_CHECKPOINT_MAX_KVTAG_SIZE (1u << 24) +#define PDC_CHECKPOINT_MAX_REGION_COUNT 1000000 +#define PDC_CHECKPOINT_MAX_HIST_NBIN 65536 // Global debug variable to control debug printfs int is_debug_g = 0; @@ -1748,7 +1748,7 @@ PDC_Server_restart(char *filename) if (key_ent == NULL || key_ent->data == NULL) PGOTO_ERROR(FAIL, "Invalid kvtag key in checkpoint"); - + int key_len = (int)strlen((char *)key_ent->data) + 1; if (key_len <= 0 || key_len > PDC_CHECKPOINT_MAX_KVTAG_KEY_LEN) PGOTO_ERROR(FAIL, "Invalid key_len %d in checkpoint", key_len); @@ -1838,15 +1838,15 @@ PDC_Server_restart(char *filename) BULKI_Entity *range_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("range", PDC_STRING)); - region_list->region_hist->range = (double *)PDC_malloc( - sizeof(double) * (size_t)nbin * 2); + region_list->region_hist->range = + (double *)PDC_malloc(sizeof(double) * (size_t)nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, sizeof(double) * (size_t)nbin * 2); BULKI_Entity *bin_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); - region_list->region_hist->bin = (uint64_t *)PDC_malloc( - sizeof(uint64_t) * (size_t)nbin); + region_list->region_hist->bin = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * (size_t)nbin); memcpy(region_list->region_hist->bin, bin_ent->data, sizeof(uint64_t) * (size_t)nbin); From 10b83df8130f70c86d8e3442f8f1045c27da0d87 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 3 Jul 2026 11:17:33 -0400 Subject: [PATCH 25/30] Validate BULKI_get results during checkpoint restart deserialization. Add null checks after every required BULKI_get() call in PDC_Server_restart() so malformed or incomplete checkpoint files fail with a clear error instead of dereferencing a null entity pointer. --- src/server/pdc_server.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 58225434b..9f986a9b0 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1620,6 +1620,8 @@ PDC_Server_restart(char *filename) // read and validate bulki version BULKI_Entity *version_entity = BULKI_get(checkpoint_bulki, BULKI_singleton_ENTITY("version_number", PDC_STRING)); + if (version_entity == NULL) + PGOTO_ERROR(FAIL, "Missing version_number in checkpoint"); int equal = BULKI_Entity_equal(version_entity, BULKI_ENTITY(PDC_CHECKPOINT_MAGIC_CURRENT, 1, PDC_STRING, PDC_CLS_ITEM)); if (!equal) { @@ -1641,6 +1643,8 @@ PDC_Server_restart(char *filename) // Extract hash key BULKI_Entity *hash_key_ent = BULKI_get(container_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + if (hash_key_ent == NULL || hash_key_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing container hash_key in checkpoint"); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); @@ -1648,6 +1652,8 @@ PDC_Server_restart(char *filename) // Extract container data BULKI_Entity *cont_data_ent = BULKI_get(container_entry, BULKI_singleton_ENTITY("cont_data", PDC_STRING)); + if (cont_data_ent == NULL || cont_data_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing container cont_data in checkpoint"); cont_entry = (pdc_cont_hash_table_entry_t *)PDC_malloc(sizeof(pdc_cont_hash_table_entry_t)); memcpy(cont_entry, cont_data_ent->data, sizeof(pdc_cont_hash_table_entry_t)); total_mem_usage_g += sizeof(pdc_cont_hash_table_entry_t); @@ -1687,6 +1693,8 @@ PDC_Server_restart(char *filename) // extract hash key BULKI_Entity *hash_key_ent = BULKI_get(hash_entry, BULKI_singleton_ENTITY("hash_key", PDC_STRING)); + if (hash_key_ent == NULL || hash_key_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing hash_key in checkpoint metadata entry"); hash_key = (uint32_t *)PDC_malloc(sizeof(uint32_t)); memcpy(hash_key, hash_key_ent->data, sizeof(uint32_t)); total_mem_usage_g += sizeof(uint32_t); @@ -1715,6 +1723,8 @@ PDC_Server_restart(char *filename) // extract metadata structure BULKI_Entity *metadata_ent = BULKI_get(metadata_obj, BULKI_singleton_ENTITY("metadata", PDC_STRING)); + if (metadata_ent == NULL || metadata_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing metadata in checkpoint"); memcpy(metadata + i, metadata_ent->data, sizeof(pdc_metadata_t)); // initialize pointers @@ -1770,6 +1780,8 @@ PDC_Server_restart(char *filename) // extract type BULKI_Entity *type_ent = BULKI_get(kvtag_entry, BULKI_singleton_ENTITY("type", PDC_STRING)); + if (type_ent == NULL) + PGOTO_ERROR(FAIL, "Missing kvtag type in checkpoint"); memcpy(&kvtag_list->kvtag->type, type_ent->data, sizeof(int8_t)); // extract value @@ -1804,11 +1816,15 @@ PDC_Server_restart(char *filename) // extract region structure BULKI_Entity *region_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("region", PDC_STRING)); + if (region_ent == NULL || region_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing region in checkpoint"); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // extract histogram flag BULKI_Entity *has_hist_ent = BULKI_get(region_entry, BULKI_singleton_ENTITY("has_hist", PDC_STRING)); + if (has_hist_ent == NULL) + PGOTO_ERROR(FAIL, "Missing has_hist in checkpoint"); int has_hist; memcpy(&has_hist, has_hist_ent->data, sizeof(int)); @@ -1824,6 +1840,8 @@ PDC_Server_restart(char *filename) BULKI_Entity *dtype_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("dtype", PDC_STRING)); + if (dtype_ent == NULL) + PGOTO_ERROR(FAIL, "Missing histogram dtype in checkpoint"); memcpy(®ion_list->region_hist->dtype, dtype_ent->data, sizeof(int)); BULKI_Entity *nbin_ent = @@ -1838,20 +1856,26 @@ PDC_Server_restart(char *filename) BULKI_Entity *range_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("range", PDC_STRING)); + if (range_ent == NULL || range_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing histogram range in checkpoint"); region_list->region_hist->range = (double *)PDC_malloc( sizeof(double) * (size_t)nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, - sizeof(double) * (size_t)nbin * 2); + sizeof(double) * (size_t)nbin * 2); BULKI_Entity *bin_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); + if (bin_ent == NULL || bin_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing histogram bin in checkpoint"); region_list->region_hist->bin = (uint64_t *)PDC_malloc( sizeof(uint64_t) * (size_t)nbin); memcpy(region_list->region_hist->bin, bin_ent->data, - sizeof(uint64_t) * (size_t)nbin); + sizeof(uint64_t) * (size_t)nbin); BULKI_Entity *incr_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING)); + if (incr_ent == NULL) + PGOTO_ERROR(FAIL, "Missing histogram incr in checkpoint"); memcpy(®ion_list->region_hist->incr, incr_ent->data, sizeof(double)); } } @@ -1937,6 +1961,8 @@ PDC_Server_restart(char *filename) // extract obj_id BULKI_Entity *obj_id_ent = BULKI_get(dataserver_obj, BULKI_singleton_ENTITY("obj_id", PDC_STRING)); + if (obj_id_ent == NULL || obj_id_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing dataserver obj_id in checkpoint"); memcpy(&new_obj_reg->obj_id, obj_id_ent->data, sizeof(uint64_t)); // extract regions @@ -1956,6 +1982,8 @@ PDC_Server_restart(char *filename) BULKI_Entity *region_ent = BULKI_get(ds_region, BULKI_singleton_ENTITY("region", PDC_STRING)); + if (region_ent == NULL || region_ent->data == NULL) + PGOTO_ERROR(FAIL, "Missing dataserver region in checkpoint"); memcpy(region_list, region_ent->data, sizeof(region_list_t)); // initialize fields (similar to above) From a4b14b792e00c1c5aedea90ba1233394acfd0350 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 3 Jul 2026 15:23:02 +0000 Subject: [PATCH 26/30] Committing clang-format changes --- src/server/pdc_server.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 0368a99f3..5e78cf0b0 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1860,10 +1860,10 @@ PDC_Server_restart(char *filename) (double *)PDC_malloc(sizeof(double) * (size_t)nbin * 2); if (range_ent == NULL || range_ent->data == NULL) PGOTO_ERROR(FAIL, "Missing histogram range in checkpoint"); - region_list->region_hist->range = (double *)PDC_malloc( - sizeof(double) * (size_t)nbin * 2); + region_list->region_hist->range = + (double *)PDC_malloc(sizeof(double) * (size_t)nbin * 2); memcpy(region_list->region_hist->range, range_ent->data, - sizeof(double) * (size_t)nbin * 2); + sizeof(double) * (size_t)nbin * 2); BULKI_Entity *bin_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("bin", PDC_STRING)); @@ -1871,10 +1871,10 @@ PDC_Server_restart(char *filename) (uint64_t *)PDC_malloc(sizeof(uint64_t) * (size_t)nbin); if (bin_ent == NULL || bin_ent->data == NULL) PGOTO_ERROR(FAIL, "Missing histogram bin in checkpoint"); - region_list->region_hist->bin = (uint64_t *)PDC_malloc( - sizeof(uint64_t) * (size_t)nbin); + region_list->region_hist->bin = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * (size_t)nbin); memcpy(region_list->region_hist->bin, bin_ent->data, - sizeof(uint64_t) * (size_t)nbin); + sizeof(uint64_t) * (size_t)nbin); BULKI_Entity *incr_ent = BULKI_get(histogram, BULKI_singleton_ENTITY("incr", PDC_STRING)); From 82a043ef4f2c598e8c99f6b8deecc9d5a2474ff2 Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 3 Jul 2026 11:30:16 -0400 Subject: [PATCH 27/30] Remove unused variable names. --- src/server/pdc_server.c | 3 +-- src/tests/tags/kvtag_get_verify_scale.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index 5e78cf0b0..d4ec2eced 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1576,7 +1576,7 @@ PDC_Server_restart(char *filename) FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; - int i, j, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; + int i, nobj = 0, all_nobj = 0, all_n_region, total_region = 0; int all_cont; pdc_metadata_t * metadata, *elt; region_list_t * region_list; @@ -1590,7 +1590,6 @@ PDC_Server_restart(char *filename) BULKI_Entity *containers_array = NULL; BULKI_Entity *metadata_entries_array = NULL; BULKI_Entity *dataserver_regions_array = NULL; - char magic[PDC_CHECKPOINT_MAGIC_LEN + 1]; FILE * file = NULL; #if defined(PDC_TIMING) || defined(ENABLE_MPI) diff --git a/src/tests/tags/kvtag_get_verify_scale.c b/src/tests/tags/kvtag_get_verify_scale.c index ab98d093d..52f624085 100644 --- a/src/tests/tags/kvtag_get_verify_scale.c +++ b/src/tests/tags/kvtag_get_verify_scale.c @@ -68,7 +68,7 @@ print_usage(char *name) int main(int argc, char *argv[]) { - pdcid_t pdc, cont_prop, cont; + pdcid_t pdc, cont; pdcid_t * obj_ids; int n_obj, n_query, my_obj, my_obj_s, n_tag, n_tag_s; int obj_1percent = 0; From 4d0843fc56de79b2913143457f55305a0747c633 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 3 Jul 2026 15:30:47 +0000 Subject: [PATCH 28/30] Committing clang-format changes --- src/server/pdc_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index d4ec2eced..b0346caf8 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -1590,7 +1590,7 @@ PDC_Server_restart(char *filename) BULKI_Entity *containers_array = NULL; BULKI_Entity *metadata_entries_array = NULL; BULKI_Entity *dataserver_regions_array = NULL; - FILE * file = NULL; + FILE * file = NULL; #if defined(PDC_TIMING) || defined(ENABLE_MPI) double start = MPI_Wtime(); From 87b27425c76092f959337f8ab2c9e95dae3bed0f Mon Sep 17 00:00:00 2001 From: Abdullah Al Raqibul Islam Date: Fri, 3 Jul 2026 11:42:06 -0400 Subject: [PATCH 29/30] Fix out-of-bounds access in BULKI_delete_incremental(). When removing the last remaining key, the function decremented numKeys and then always copied from index numKeys - 1. With numKeys equal to 0, that index underflows and can corrupt memory or crash. Skip the swap when the deleted key is already at the last index. Also reject NULL bulki or key arguments at entry. --- src/commons/serde/bulki/bulki.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index d8966e55f..10ae9ced2 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -659,15 +659,23 @@ BULKI_delete_incremental(BULKI *bulki, BULKI_Entity *key) { FUNC_ENTER(NULL); + if (bulki == NULL || key == NULL) { + LOG_ERROR("Error: bulki or key is NULL in BULKI_delete_incremental()\n"); + FUNC_LEAVE(NULL); + } + BULKI_Entity *value = NULL; + size_t last_index = bulki->numKeys - 1; for (size_t i = 0; i < bulki->numKeys; i++) { if (BULKI_Entity_equal(&bulki->header->keys[i], key)) { value = &bulki->data->values[i]; bulki->header->headerSize -= key->size; bulki->data->dataSize -= value->size; bulki->numKeys--; - memcpy(&bulki->header->keys[i], &bulki->header->keys[bulki->numKeys - 1], sizeof(BULKI_Entity)); - memcpy(&bulki->data->values[i], &bulki->data->values[bulki->numKeys - 1], sizeof(BULKI_Entity)); + if (i != last_index) { + memcpy(&bulki->header->keys[i], &bulki->header->keys[bulki->numKeys - 1], sizeof(BULKI_Entity)); + memcpy(&bulki->data->values[i], &bulki->data->values[bulki->numKeys - 1], sizeof(BULKI_Entity)); + } break; } } From 00b990f189f9fec34fe9e92cb7f48072b37a60c2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 3 Jul 2026 15:42:34 +0000 Subject: [PATCH 30/30] Committing clang-format changes --- src/commons/serde/bulki/bulki.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/commons/serde/bulki/bulki.c b/src/commons/serde/bulki/bulki.c index 10ae9ced2..2e9e234cd 100644 --- a/src/commons/serde/bulki/bulki.c +++ b/src/commons/serde/bulki/bulki.c @@ -664,8 +664,8 @@ BULKI_delete_incremental(BULKI *bulki, BULKI_Entity *key) FUNC_LEAVE(NULL); } - BULKI_Entity *value = NULL; - size_t last_index = bulki->numKeys - 1; + BULKI_Entity *value = NULL; + size_t last_index = bulki->numKeys - 1; for (size_t i = 0; i < bulki->numKeys; i++) { if (BULKI_Entity_equal(&bulki->header->keys[i], key)) { value = &bulki->data->values[i]; @@ -673,8 +673,10 @@ BULKI_delete_incremental(BULKI *bulki, BULKI_Entity *key) bulki->data->dataSize -= value->size; bulki->numKeys--; if (i != last_index) { - memcpy(&bulki->header->keys[i], &bulki->header->keys[bulki->numKeys - 1], sizeof(BULKI_Entity)); - memcpy(&bulki->data->values[i], &bulki->data->values[bulki->numKeys - 1], sizeof(BULKI_Entity)); + memcpy(&bulki->header->keys[i], &bulki->header->keys[bulki->numKeys - 1], + sizeof(BULKI_Entity)); + memcpy(&bulki->data->values[i], &bulki->data->values[bulki->numKeys - 1], + sizeof(BULKI_Entity)); } break; }