diff --git a/.gitignore b/.gitignore index 0ccf616c8..68890fddc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,16 @@ # ignore swp files *.swp +.cache # exes *.exe - *.pdf - src/install # ignore vscode files .vscode +# ignore jetbrains +.idea # ignore intellij files .idea diff --git a/CMakeLists.txt b/CMakeLists.txt index 67022035e..75245174d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,143 @@ foreach(p endif() endforeach() +# Dependencies directories +# ZFP +if(NOT DEFINED ENV{ZFP_ROOT_DIR} OR "$ENV{ZFP_ROOT_DIR}" STREQUAL "") + message(FATAL_ERROR "Environment variable ZFP_ROOT_DIR is not set") +endif() +set(ZFP_ROOT $ENV{ZFP_ROOT_DIR}) +# JSON-C +if(NOT DEFINED ENV{JSONC_ROOT_DIR} OR "$ENV{JSONC_ROOT_DIR}" STREQUAL "") + message(FATAL_ERROR "Environment variable JSONC_ROOT_DIR is not set") +endif() +set(JSONC_ROOT $ENV{JSONC_ROOT_DIR}) +# libsodium +if(NOT DEFINED ENV{LIBSODIUM_ROOT_DIR} OR "$ENV{LIBSODIUM_ROOT_DIR}" STREQUAL "") + message(FATAL_ERROR "Environment variable LIBSODIUM_ROOT_DIR is not set") +endif() +set(LIBSODIUM_ROOT $ENV{LIBSODIUM_ROOT_DIR}) +if(DEFINED ENV{CUDA_ROOT_DIR} AND NOT "$ENV{CUDA_ROOT_DIR}" STREQUAL "") + set(CUDA_ROOT $ENV{CUDA_ROOT_DIR}) +endif() + +# ZFP +find_path(ZFP_INCLUDE_DIR NAMES zfp.h PATHS "${ZFP_ROOT}/include") +find_library(ZFP_LIBRARY NAMES zfp PATHS "${ZFP_ROOT}/lib64" "${ZFP_ROOT}/lib") +if(NOT ZFP_INCLUDE_DIR) + message(FATAL_ERROR "Could not find ZFP headers in ${ZFP_ROOT}/include") +endif() +if(NOT ZFP_LIBRARY) + message(FATAL_ERROR "Could not find ZFP library in ${ZFP_ROOT}/lib64 or ${ZFP_ROOT}/lib") +endif() +include_directories(${ZFP_INCLUDE_DIR}) +link_libraries(${ZFP_LIBRARY}) + +# JSONC +find_path(JSONC_INCLUDE_DIR NAMES json-c/json.h PATHS "${JSONC_ROOT}/include") +find_library(JSONC_LIBRARY NAMES json-c PATHS "${JSONC_ROOT}/lib64" "${JSONC_ROOT}/lib") +if(NOT JSONC_INCLUDE_DIR) + message(FATAL_ERROR "Could not find JSON-C headers in ${JSONC_ROOT}/include") +endif() +if(NOT JSONC_LIBRARY) + message(FATAL_ERROR "Could not find JSON-C library in ${JSONC_ROOT}/lib64 or ${LIBSODIUM_ROOT}/lib") +endif() +include_directories(${JSONC_INCLUDE_DIR}) +link_libraries(${JSONC_LIBRARY}) + +# LIBSODIUM +find_path(LIBSODIUM_INCLUDE_DIR NAMES sodium.h PATHS "${LIBSODIUM_ROOT}/include") +find_library(LIBSODIUM_LIBRARY NAMES sodium PATHS "${LIBSODIUM_ROOT}/lib") +if(NOT LIBSODIUM_INCLUDE_DIR) + message(FATAL_ERROR "Could not find libsodium headers in ${LIBSODIUM_ROOT}/include") +endif() +if(NOT LIBSODIUM_LIBRARY) + message(FATAL_ERROR "Could not find libsodium library in ${LIBSODIUM_ROOT}/lib") +endif() +include_directories(${LIBSODIUM_INCLUDE_DIR}) +link_libraries(${LIBSODIUM_LIBRARY}) + +# CUDA +if(CUDA_ROOT) + # Find include directory + find_path(CUDA_INCLUDE_DIR NAMES cuda_runtime.h PATHS "${CUDA_ROOT}/include") + # Find runtime library + find_library(CUDA_RUNTIME_LIBRARY NAMES cudart PATHS "${CUDA_ROOT}/lib64" "${CUDA_ROOT}/lib") + if(NOT CUDA_INCLUDE_DIR) + message(FATAL_ERROR "Could not find CUDA headers in ${CUDA_ROOT}/include") + endif() + if(NOT CUDA_RUNTIME_LIBRARY) + message(FATAL_ERROR "Could not find CUDA runtime library in ${CUDA_ROOT}/lib64 or ${CUDA_ROOT}/lib") + endif() + include_directories(${CUDA_INCLUDE_DIR}) + link_libraries(${CUDA_RUNTIME_LIBRARY}) + set(CUDA_ENABLED 1 CACHE BOOL "Enable CUDA globally") + if(CUDA_ENABLED) + add_compile_definitions(CUDA_ENABLED=1) + add_definitions(-DUSE_CUDA) + endif() +endif() + +if(NOT DEFINED SZ_CPU_DIR) + set(SZ_CPU_DIR $ENV{SZ_CPU_DIR}) +endif() +message(STATUS "SZ_CPU_DIR = ${SZ_CPU_DIR}") +# SZ3 +find_path(SZ3_INCLUDE_DIR NAMES sz3c.h PATHS "${SZ_CPU_DIR}/include/SZ3c") +find_library(SZ3_LIBRARY NAMES SZ3c PATHS "${SZ_CPU_DIR}/lib64" "${SZ_CPU_DIR}/lib") +if(NOT SZ3_INCLUDE_DIR) + message(FATAL_ERROR "Could not find SZ3 headers in ${SZ_CPU_DIR}/include") +endif() +if(NOT SZ3_LIBRARY) + message(FATAL_ERROR "Could not find SZ3 library in ${SZ_CPU_DIR}/lib64 or ${SZ_CPU_DIR}/lib") +endif() +include_directories(${SZ3_INCLUDE_DIR}) +link_libraries(${SZ3_LIBRARY}) +# Check if CUSZ_GPU_DIR is defined; if not, take from environment +if(NOT DEFINED CUSZ_GPU_DIR) + set(CUSZ_GPU_DIR $ENV{CUSZ_GPU_DIR}) +endif() +# GPU CUSZ +if(NOT DEFINED CUSZ_GPU_DIR) + set(CUSZ_GPU_DIR $ENV{CUSZ_GPU_DIR}) +endif() +message(STATUS "CUSZ_GPU_DIR = ${CUSZ_GPU_DIR}") +# Find CUSZ headers (not SZ3) +find_path(CUSZ_INCLUDE_DIR + NAMES cusz.h context.h api_v2.h header.h + PATHS "${CUSZ_GPU_DIR}/include/cusz" +) +if(NOT CUSZ_INCLUDE_DIR) + message(FATAL_ERROR "Could not find CUSZ headers in ${CUSZ_GPU_DIR}/include/cusz") +endif() +include_directories(${CUSZ_INCLUDE_DIR}) +find_library(CUSZ_LIBRARY + NAMES cusz + PATHS + "${CUSZ_GPU_DIR}/lib" + "${CUSZ_GPU_DIR}/lib64" +) +if(NOT CUSZ_LIBRARY) + message(FATAL_ERROR "Could not find CUSZ library in ${CUSZ_GPU_DIR}/lib or lib64") +endif() +message(STATUS "CUSZ_LIBRARY = ${CUSZ_LIBRARY}") +link_libraries(${CUSZ_LIBRARY}) + +# JSONC +if(NOT DEFINED TURBO_DIR) + set(TURBO_DIR $ENV{TURBO_DIR}) +endif() +find_path(TURBO_DIR_INCLUDE NAMES ic.h PATHS "${TURBO_DIR}/include") +find_library(TURBO_DIR_LIB NAMES ic PATHS "${TURBO_DIR}") +if(NOT TURBO_DIR_INCLUDE) + message(FATAL_ERROR "Could not find TURBO_DIR headers in ${TURBO_DIR}/include") +endif() +if(NOT TURBO_DIR_LIB) + message(FATAL_ERROR "Could not find TURBO_DIR library in ${TURBO_DIR}") +endif() +include_directories(${TURBO_DIR_INCLUDE}) +link_libraries(${TURBO_DIR_LIB}) + # Set a consistent MACOSX_RPATH default across all CMake versions. # checks if the variable CMAKE_MACOSX_RPATH is not defined and sets it to 0 if it is not defined. # The CMAKE_MACOSX_RPATH variable is used to control the behavior of the macOS-specific RPATH feature in CMake. @@ -394,10 +531,14 @@ endif() option(PDC_SERVER_CACHE "Enable Server Caching." ON) if(PDC_SERVER_CACHE) set(PDC_SERVER_CACHE 1) - set(PDC_SERVER_CACHE_MAX_GB "32" CACHE STRING "Max GB for server cache") - set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "3" CACHE STRING "Idle time to initiate flush from server cache") + set(PDC_SERVER_CACHE_MAX_GB "64" CACHE STRING "Max GB for server cache") + set(PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB "32" CACHE STRING "Max GB for server transformation cache") + set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "2" CACHE STRING "Idle time to initiate flush from server cache") - add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME}) + add_compile_definitions( + PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB=${PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB} + PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} + PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME}) endif() @@ -419,7 +560,16 @@ if(PDC_ENABLE_APP_CLOSE_SERVER) endif() #----------------------------------------------------------------------------- -# +# Path to graphs for tests +#----------------------------------------------------------------------------- +set(TF_GRAPHS_DIR "${CMAKE_SOURCE_DIR}/tf_client/graphs/") +if(NOT EXISTS "${TF_GRAPHS_DIR}") + message(FATAL_ERROR "Directory ${TF_GRAPHS_DIR} does not exist!") +endif() +add_compile_definitions(TF_GRAPHS_DIR="${TF_GRAPHS_DIR}") + +#----------------------------------------------------------------------------- +# Wait for data fanalized in FS when object unmap is called #----------------------------------------------------------------------------- option(PDC_ENABLE_WAIT_DATA "Wait for data fanalized in FS when object unmap is called." OFF) diff --git a/docs/source/data_flyway.rst b/docs/source/data_flyway.rst new file mode 100644 index 000000000..e69de29bb diff --git a/docs/source/index.rst b/docs/source/index.rst index 4c459dcb0..b57145a80 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,22 +1,19 @@ Proactive Data Containers (PDC) =============================== - Welcome to the documentation for **Proactive Data Containers (PDC)** — a high-performance data management framework designed to support flexible, scalable, and efficient data movement and access across diverse memory and storage systems. - Whether you're a new user, a developer integrating PDC into your application, or a contributor to the project, this documentation provides the resources you need to get started and dive deeper. - -Here’s how to navigate: - +Here's how to navigate: - **Getting Started** Learn the basics of PDC, including core concepts, architecture, and key services. Start here if you're new to PDC. - **Developer & API Guide** Explore the client API, internal design considerations, and tools to help you develop and debug PDC-based applications. +- **Transformation Framework** + Learn how to attach compression, encryption, and custom transformation pipelines to PDC objects using directed graphs. - **Community and Contributions** Find guidelines for contributing to PDC and additional reference material in the appendix. - Use the table of contents below to explore the sections in more detail .. toctree:: @@ -34,9 +31,15 @@ Use the table of contents below to explore the sections in more detail client_api client_tools +.. toctree:: + :maxdepth: 2 + :caption: Data Flyway + + pdc_tf_docs + .. toctree:: :maxdepth: 1 :caption: Community and Contributions contributing - roadmap + roadmap \ No newline at end of file diff --git a/docs/source/pdc_tf_docs.rst b/docs/source/pdc_tf_docs.rst new file mode 100644 index 000000000..ff9d719c1 --- /dev/null +++ b/docs/source/pdc_tf_docs.rst @@ -0,0 +1,1025 @@ +PDC Transformation Framework +============================= + +The PDC Transformation Framework (PDC TF) is a server-side pipeline system that allows +data regions to be automatically transformed (compressed, encrypted, etc.) as they flow +between clients and the PDC data server. Transformations are described as **directed graphs** +where nodes are named **states** (e.g. ``"raw"``, ``"compressed"``) and edges are **functions** +that convert data from one state to another. The framework supports both CPU and GPU execution +and can dynamically schedule work across available devices based on real-time utilization metrics. + +Here's how to navigate: + +- **Architecture & Core Concepts** + Understand how states, functions, regions, and directed graphs fit together. + +- **JSON Graph Format** + Learn how to describe transformation pipelines in JSON. + +- **Client & Server API** + Reference for attaching graphs to objects and executing transformations. + +- **Scheduler & Profiler** + How the framework selects between CPU and GPU edges at runtime. + +- **Builtin Transformations** + Descriptions of every builtin compression and encryption function. + +- **Extending the Framework** + How to add your own external transformation functions. + +Use the table of contents below to explore each section in detail. + +.. toctree:: + :maxdepth: 2 + :caption: Architecture & Core Concepts + + tf_architecture + tf_core_concepts + +.. toctree:: + :maxdepth: 2 + :caption: Configuration & Graph Format + + tf_json_format + tf_compile_flags + +.. toctree:: + :maxdepth: 2 + :caption: Client & Server API + + tf_client_api + tf_server_api + tf_parameter_macros + +.. toctree:: + :maxdepth: 2 + :caption: Scheduler & Profiler + + tf_scheduler + tf_profiler + +.. toctree:: + :maxdepth: 2 + :caption: Builtin Transformations + + tf_builtin_registry + tf_builtin_zfp_cpu + tf_builtin_zfp_gpu + tf_builtin_sz_cpu + tf_builtin_sz_gpu + tf_builtin_encrypt + tf_builtin_turbo + +.. toctree:: + :maxdepth: 1 + :caption: Extending the Framework + + tf_external_transforms + tf_contributing + + +------------------------------------------------------------------------------- + +.. _tf_architecture: + +Architecture Overview +===================== + +.. code-block:: text + + Client Write Server Store + ───────────────────────────────────────────────────────────────── + [raw data] [transformed data] + │ ▲ + │ client_state = "raw" │ store_state = "compressed" + ▼ │ + PDCtf_attach_to_region/obj PDCtf_exec_graph + │ │ + └──────── Directed Graph ────────────────┘ + (JSON file) + states: raw ──zfp_compress──▶ compressed + ──sz_compress───▶ compressed + +On **write**: data arrives in ``client_state``, the graph is traversed to reach ``store_state``, +and the transformed data is written to disk. + +On **read**: data is read from disk in ``store_state``, the graph is traversed in reverse to +reach ``client_state``, and the raw data is returned to the client. + +Data Flow +--------- + +Each edge in the graph is a transformation function. When the server needs to move data +from one state to another, it finds the shortest path through the directed graph and +executes each edge's function in sequence. If multiple edges exist between the same two +nodes (e.g. both a CPU and GPU compressor), the scheduler picks the best one at runtime. + +The data buffer (``void **region_data``) and its shape descriptor (``pdc_tf_region_t``) are +updated in place as data flows through the pipeline. After compression, for example, the +region becomes a flat 1D byte array — subsequent functions in the chain see that new shape. + + +------------------------------------------------------------------------------- + +.. _tf_core_concepts: + +Core Concepts +============= + +States +------ + +A state is a named node in the directed graph representing a particular data representation. +Examples: ``"raw"``, ``"compressed"``, ``"encrypted"``. + +.. code-block:: c + + typedef struct pdc_tf_state_t { + char *name; // State name, e.g. "raw" + PDC_VECTOR *pdc_tf_dg_params_vector; // Per-region parameter storage + } pdc_tf_state_t; + +Functions +--------- + +A function is a directed edge between two states. It wraps a C function pointer and carries +metadata about which device it runs on and where it is located (builtin vs. external shared library). + +.. code-block:: c + + typedef struct pdc_tf_func_t { + pdc_tf_dev_t dev; // PDC_TF_CPU_DEVICE or PDC_TF_GPU_DEVICE + pdc_tf_location_t location; // PDC_TF_BUILTIN or PDC_TF_EXTERNAL + char *name; // e.g. "zfp_compress" + char *params_str; // Optional JSON params string from graph file + c_func_t c_func; // The actual function pointer + PDC_VECTOR *pdc_tf_dg_params_vector; // Per-region output params storage + } pdc_tf_func_t; + +Regions +------- + +A region describes the shape and element type of a data buffer as it passes through the pipeline. +Each transformation receives the input region descriptor and must populate the output region +descriptor if the shape or type changes. + +.. code-block:: c + + typedef struct pdc_tf_region_t { + size_t ndim; // Number of dimensions (1–4) + uint8_t pdc_var_type; // PDC_FLOAT, PDC_DOUBLE, PDC_INT32, etc. + uint64_t size[DIM_MAX]; // Per-dimension element counts + } pdc_tf_region_t; + +After compression the output region always becomes:: + + { ndim=1, pdc_var_type=PDC_CHAR, size[0]=compressed_bytes } + +Transform Function Signature +----------------------------- + +Every builtin and user-defined external transform function must match this prototype: + +.. code-block:: c + + typedef bool (*c_func_t)( + pdc_tf_internal_param internal_param, // Graph handle + flat offset + char *params_str, // Optional params string from JSON + void **region_data, // Pointer to data buffer (may be replaced) + pdc_tf_region_t input_region, // Describes input data shape/type + pdc_tf_region_t *output_region // Must be populated if shape/type changes + ); + +**Contract:** + +- ``*region_data`` may be replaced with a newly ``malloc``'d buffer. + The old pointer must **not** be freed by the function — the framework handles that. +- ``output_region`` is pre-initialized to ``input_region``. + Only update it if the transformation changes the shape or type. +- Return ``true`` on success, ``false`` on failure. + +Per-Region Parameters +--------------------- + +Because multiple data regions may flow through the same graph concurrently, each function +and state node stores parameters keyed by a **flat conceptual offset** — a scalar that +uniquely identifies which region the parameters belong to. + +.. code-block:: c + + typedef struct pdc_tf_dg_params_t { + uint64_t flat_conceptual_offset; // Unique region key + void *params; // Pointer to parameter struct + uint64_t params_size; // Size of params in bytes + } pdc_tf_dg_params_t; + +Functions use the ``SET_FUNC_PARAMS`` / ``GET_FUNC_PARAMS`` macros to store and retrieve these. +For example, ``zfp_compress`` stores the original region descriptor so ``zfp_decompress`` knows +the decompressed shape when it is later called. + + +------------------------------------------------------------------------------- + +.. _tf_json_format: + +JSON Graph File Format +====================== + +Graphs are described in JSON and loaded at runtime via :c:func:`PDCtf_dg_json_create`. + +Schema +------ + +.. code-block:: json + + { + "name": "my_graph", + "lib_path": "/optional/path/to/external.so", + "states": [ + { "name": "raw" }, + { "name": "compressed" } + ], + "functions": [ + { + "name": "zfp_compress", + "device": "CPU", + "location": "builtin", + "input_state": "raw", + "output_state": "compressed", + "params": "{\"rate\": 8}" + }, + { + "name": "zfp_decompress", + "device": "CPU", + "location": "builtin", + "input_state": "compressed", + "output_state": "raw" + } + ] + } + +Fields +------ + +.. list-table:: + :header-rows: 1 + :widths: 20 10 70 + + * - Field + - Required + - Description + * - ``name`` + - Yes + - Graph name (informational). + * - ``lib_path`` + - No + - Path to a shared library for external functions. Required if any function has ``"location": "external"``. + * - ``states`` + - Yes + - Array of named state nodes. Each entry requires a ``"name"`` field. + * - ``functions`` + - Yes + - Array of edges. See function fields below. + +Function Fields +--------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 10 70 + + * - Field + - Required + - Description + * - ``name`` + - Yes + - Name of the function. Must match a registered builtin or an exported symbol in ``lib_path``. + * - ``device`` + - Yes + - ``"CPU"`` or ``"GPU"``. + * - ``location`` + - Yes + - ``"builtin"`` or ``"external"``. + * - ``input_state`` + - Yes + - Name of the source state node. + * - ``output_state`` + - Yes + - Name of the destination state node. + * - ``params`` + - No + - Optional string passed as ``params_str`` to the function at runtime. + +Parallel Edges +-------------- + +Multiple functions between the same two states are fully supported. The scheduler will +select the best candidate at runtime based on device utilization and historical timing. +This allows you to register both a CPU and a GPU version of a compressor and let the +framework decide which to use: + +.. code-block:: json + + { "name": "zfp_compress", "device": "CPU", "location": "builtin", + "input_state": "raw", "output_state": "compressed" }, + { "name": "zfp_compress_cuda", "device": "GPU", "location": "builtin", + "input_state": "raw", "output_state": "compressed" } + + +------------------------------------------------------------------------------- + +.. _tf_compile_flags: + +Compile-Time Feature Flags +=========================== + +The following CMake flags control which builtin transformations are compiled in. +They are currently also ``#define``'d at the top of ``pdc_tf_builtin_common.h`` +as a temporary workaround (marked ``FIXME`` — these should be driven solely by CMake). + +.. list-table:: + :header-rows: 1 + :widths: 40 60 + + * - Flag + - Effect + * - ``ENABLE_TF_ZFP_COMPRESSION`` + - Enables ZFP CPU compress/decompress + * - ``CUDA_ENABLED`` + - Enables ZFP GPU compress/decompress (requires ZFP built with CUDA support) + * - ``ENABLE_TF_SZ_COMPRESSION`` + - Enables SZ3 CPU compress/decompress + * - ``ENABLE_TF_SZ_GPU_COMPRESSSION`` + - Enables cuSZ GPU compress/decompress + * - ``ENABLE_TF_SECRET_BOX_ENCRYPTION`` + - Enables libsodium SecretBox encrypt/decrypt + * - ``ENABLE_TF_TURBO_COMPRESSION`` + - Enables TurboPFor integer compress/decompress + + +------------------------------------------------------------------------------- + +.. _tf_client_api: + +Client API +========== + +Declared in ``pdc_tf.h``. These functions are called from client applications to +attach transformation graphs to PDC objects and regions. + +Initialization +-------------- + +.. c:function:: perr_t PDCtf_init() + + Must be called once during PDC initialization. Registers the ``PDC_TF_DG`` type + and populates the builtin function registry by calling :c:func:`PDCtf_init_builtin_funcs`. + +Graph Loading +------------- + +.. c:function:: pdcid_t PDCtf_dg_json_create(char *json_filepath) + + Parses a JSON graph file and returns a handle (``pdcid_t``) to the in-memory directed + graph. External functions referenced in the JSON are loaded via ``dlopen``/``dlsym`` + using the ``lib_path`` field. Returns ``0`` on failure. + +.. c:function:: perr_t PDCtf_close_dg(pdcid_t dg_id) + + Frees all resources associated with a directed graph, including all state and function + nodes and their per-region parameter storage. + +Attaching Graphs +---------------- + +.. c:function:: perr_t PDCtf_attach_to_region(pdcid_t dg_id, pdcid_t obj_id, pdcid_t remote_reg, char *client_state, char *store_state) + + Attaches a transformation graph to a **specific region** of an object. Only transfers + touching that exact region (matched by ndim, offset, size, and element type) will be + routed through the graph. + + :param dg_id: Handle to the directed graph returned by :c:func:`PDCtf_dg_json_create`. + :param obj_id: PDC object to attach the graph to. + :param remote_reg: Region ID describing the specific region. + :param client_state: The state name representing data as the client sees it (e.g. ``"raw"``). + :param store_state: The state name representing data as stored on disk (e.g. ``"compressed"``). + +.. c:function:: perr_t PDCtf_attach_to_obj(pdcid_t dg_id, pdcid_t obj_id, char *client_state, char *store_state) + + Attaches a transformation graph to **all regions** of an object. Every read/write on + the object will be routed through the graph regardless of region. + +.. c:function:: perr_t PDCtf_attach_to_objs(pdcid_t dg_id, pdcid_t *obj_ids, int num_ids, char *client_state, char *store_state) + + Convenience wrapper that calls :c:func:`PDCtf_attach_to_obj` for each object in the array. + +Diagnostics +----------- + +.. c:function:: void PDCtf_print_dg(pdcid_t dg_id, bool write_to_file) + + Prints the graph in Graphviz DOT format. CPU edges are blue, GPU edges are red. + If ``write_to_file`` is ``true``, output goes to ``graph.txt``; otherwise stdout. + +.. c:function:: void PDCtf_print_exec_path(pdcid_t dg_id, char *cur_state, char *desired_state) + + Prints the shortest path through the graph between two named states, showing + which transformation functions would be executed in order. + + +------------------------------------------------------------------------------- + +.. _tf_server_api: + +Server API +========== + +Declared in ``pdc_tf_server.h``. These functions are used internally by the PDC +data server and are not part of the client-facing API. + +.. c:function:: perr_t PDCtf_store_json_mapping(pdcid_t obj_id, char *json_filepath, char *cur_state, char *client_state, char *store_state, uint64_t *offset, uint64_t *size, uint8_t ndim, pdc_var_type_t pdc_var_type) + + Called on the server when a client attaches a graph to a region. Loads the JSON graph, + creates the in-memory directed graph if it does not yet exist for this object, and stores + the conceptual region → actual region mapping. + + If the object already has a graph attached, the filepath is validated to match. If the + same region offset is attached twice the existing mapping is overwritten. + +.. c:function:: perr_t PDCtf_exec_graph(pdc_dg_t *dg, uint64_t flat_conceptual_offset, char *cur_state, char *desired_state, pdc_tf_region_t input_region, pdc_tf_region_t *output_region, void **input, int is_write) + + Runs the shortest path through the graph from ``cur_state`` to ``desired_state``, + executing each edge's transformation function in sequence. When multiple parallel edges + exist between the same two nodes, the scheduler selects the best one (see :ref:`tf_scheduler`). + + Updates ``*input`` and ``*output_region`` in place as data flows through the pipeline. + Intermediate buffers that are replaced during the pipeline are freed automatically, + except for the very first input buffer when ``is_write`` is true (which is owned by + the caller). + + :param dg: The directed graph to traverse. + :param flat_conceptual_offset: Scalar key identifying which region's parameters to use. + :param cur_state: Starting state name. + :param desired_state: Target state name. + :param input_region: Shape/type of the input data. + :param output_region: Populated with the shape/type of the final output. + :param input: Double pointer to the data buffer; updated as transforms run. + :param is_write: Non-zero if this is a write (client→server) operation. + + +------------------------------------------------------------------------------- + +.. _tf_parameter_macros: + +Parameter Macros +================ + +These macros are used **inside transformation functions** to persist state between +compression and decompression passes. For example, ``zfp_compress`` uses +``SET_FUNC_PARAMS`` to save the original region shape so that ``zfp_decompress`` +can restore it later. + +Each call is automatically scoped to the current region via the ``flat_conceptual_offset`` +embedded in ``internal_param``, so concurrent regions flowing through the same graph +do not interfere with each other. + +Function Parameter Macros +-------------------------- + +.. code-block:: c + + // Store parameters for a named function edge + SET_FUNC_PARAMS("func_name", PDC_TF_CPU_DEVICE, params_ptr, sizeof(params_t)); + + // Retrieve parameters previously stored for a named function edge + GET_FUNC_PARAMS("func_name", PDC_TF_CPU_DEVICE, (void **)¶ms_ptr, ¶ms_size); + +State Parameter Macros +----------------------- + +.. code-block:: c + + // Store parameters for a named state node + SET_STATE_PARAMS("state_name", params_ptr, sizeof(params_t)); + + // Retrieve parameters previously stored for a named state node + GET_STATE_PARAMS("state_name", (void **)¶ms_ptr, ¶ms_size); + +Usage Pattern +------------- + +The typical pattern for a compress/decompress pair is: + +.. code-block:: c + + // In the compress function — save the original region so decompress can restore it + my_params_t *out_params = malloc(sizeof(my_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("my_compress", PDC_TF_CPU_DEVICE, out_params, sizeof(my_params_t)); + + // In the decompress function — retrieve the saved region + my_params_t *in_params = NULL; + uint64_t in_params_size; + GET_FUNC_PARAMS("my_compress", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + PDCtf_copy_tf_region_t(&in_params->decompressed_region, output_region); + + +------------------------------------------------------------------------------- + +.. _tf_scheduler: + +Scheduler +========= + +When multiple edges exist between the same two states (e.g. both a CPU and a GPU +``zfp_compress``), :c:func:`PDCtf_exec_graph` selects one using the following priority: + +Selection Algorithm +------------------- + +1. **Force GPU** — If the environment variable ``USE_GPU`` is set, or if ``close_time_g`` + is set (indicating the server is approaching a deadline), always pick the GPU edge + on device 0. + +2. **Skip overloaded GPU** — If the least-loaded GPU has average utilization above 50%, + skip all GPU edges and fall back to CPU. + +3. **Expected time** — For each remaining candidate edge, compute: + + .. math:: + + \text{expected\_time} = \frac{\text{avg\_past\_time}}{\max(0.1,\ 1.0 - \text{device\_utilization})} + + The edge with the lowest expected time is selected. + +Execution Time History +----------------------- + +After each transformation, the actual elapsed wall-clock time is scaled by the +device's free capacity to produce a *projected free-device time*: + +.. code-block:: text + + projected_time = transform_time * (1.0 - max(device_utilization, 0.1)) + +This projected time is stored in a rolling history buffer (``prev_cpu_times`` / +``prev_gpu_times``, length ``NUM_TF_FUNC_TIMES = 5``) and used to compute +``avg_past_time`` in future scheduling decisions. + +Logging +------- + +When running as server rank 0, the scheduler emits ``LOG_WARNING`` lines prefixed +with ``SCHED:`` describing utilization, candidate edge evaluation, and the final +selection. These can be used to diagnose scheduling behavior in production. + + +------------------------------------------------------------------------------- + +.. _tf_profiler: + +Profiler +======== + +The profiler maintains rolling buffers of recent GPU and CPU utilization samples +that feed into the scheduler's device selection logic. + +Declared in ``pdc_tf_profiler.h``, implemented in ``pdc_tf_profiler.c``. + +Rolling Buffer +-------------- + +All samples are stored in a fixed-size rolling buffer of length ``PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE = 3``. +Older samples are overwritten as new ones arrive. Averages are computed over all +non-null entries in the buffer. + +GPU Profiling (NVML) +-------------------- + +Uses the NVIDIA Management Library (NVML) to sample per-device utilization. +On the first call, NVML is initialized and the device count is queried. Subsequent +calls populate a per-device sample array containing: + +- ``gpu_utilization`` — GPU compute utilization as a fraction (0.0–1.0). +- ``memory_utilization`` — Memory controller utilization percentage. +- ``memory_total / memory_used / memory_free`` — Absolute memory figures in bytes. + +CPU Profiling +------------- + +CPU utilization is estimated from the server's own event loop timing: + +.. code-block:: text + + cpu_utilization = 1.0 - (elapsed_progress_time / elapsed_total_time) + +Where ``elapsed_progress_time`` is the time the server spent doing useful work +and ``elapsed_total_time`` is the total wall time of the loop iteration. This +approximates the fraction of time the CPU was busy. + +API +--- + +.. c:function:: perr_t pdc_tf_update_profiler(double elapsed_total_time_sec, double elapsed_progress_time_sec) + + Call from the server event loop to push a new CPU sample and refresh GPU samples. + Both the NVML and CPU profilers are updated in a single call. + +.. c:function:: double pdc_tf_avg_gpu_utilization(unsigned int device_index) + + Returns the rolling average GPU utilization (0.0–1.0) for the given device index. + Returns ``-1.0`` if the device index is out of range. + +.. c:function:: double pdc_tf_avg_cpu_utilization() + + Returns the rolling average CPU utilization (0.0–1.0) across recent samples. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_registry: + +Builtin Function Registry +========================== + +The registry is a global ``PDC_VECTOR`` (``pdc_tf_builtin_funcs_vector_g``) of +``pdc_tf_builtin_func_t`` entries, each associating a ``(name, device)`` pair with +a function pointer. + +.. code-block:: c + + typedef struct pdc_tf_builtin_func_t { + char *name; // e.g. "zfp_compress" + pdc_tf_dev_t dev; // PDC_TF_CPU_DEVICE or PDC_TF_GPU_DEVICE + c_func_t c_func; // Function pointer + } pdc_tf_builtin_func_t; + +Functions are unique by ``(name, device)`` pair. This allows the same logical +operation (e.g. ``"zfp_compress"``) to have both a CPU and a GPU implementation +registered simultaneously, enabling the scheduler to choose between them. + +Lifecycle +--------- + +1. :c:func:`PDCtf_init` calls :c:func:`PDCtf_init_builtin_funcs` at startup. +2. :c:func:`PDCtf_init_builtin_funcs` calls :c:func:`PDCtf_add_builtin_func` for + each enabled builtin, conditioned on compile-time feature flags. +3. During graph loading (:c:func:`PDCtf_dg_json_create_common`), each JSON function + entry is bound to its registered function pointer via :c:func:`PDCtf_link_builtin_func`. + +API +--- + +.. c:function:: perr_t PDCtf_add_builtin_func(char *func_name, c_func_t c_func, pdc_tf_dev_t dev) + + Registers a single function pointer in the global builtin registry. + +.. c:function:: perr_t PDCtf_link_builtin_func(char *func_name, pdc_tf_dev_t dev, pdc_tf_func_t *f) + + Searches the registry for ``(func_name, dev)`` and sets ``f->c_func`` on success. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_zfp_cpu: + +ZFP Compression — CPU +===================== + +**Source file:** ``pdc_tf_builtin_zfp_cpu.c`` + +**Requires:** ``ENABLE_TF_ZFP_COMPRESSION`` + +ZFP is a floating-point and integer array compression library optimized for +multidimensional scientific data. The CPU implementation uses the standard ZFP +bitstream API in fixed-rate mode. + +Supported Types +--------------- + +``PDC_FLOAT``, ``PDC_DOUBLE``, ``PDC_INT32``, ``PDC_INT64``. +Supports 1D–4D arrays. + +Rate +---- + +The rate is set to ``8 * sizeof(element)`` bits/value (``FIXED_CR_RATIO = 1``). +This corresponds to lossless-equivalent throughput at the ZFP bitstream level. + +pdc_tf_builtin_zfp_compress +----------------------------- + +- Allocates a ZFP field descriptor matching the input region shape and type. +- Opens a bitstream over a ``PDC_malloc``'d buffer sized by ``zfp_stream_maximum_size``. +- Calls ``zfp_compress`` and replaces ``*region_data`` with the compressed buffer. +- Sets output region to ``{ndim=1, pdc_var_type=PDC_CHAR, size[0]=compressed_bytes}``. +- Saves the original input region via ``SET_FUNC_PARAMS("zfp_compress", PDC_TF_CPU_DEVICE, ...)``. + +pdc_tf_builtin_zfp_decompress +------------------------------- + +- Retrieves the saved region descriptor via ``GET_FUNC_PARAMS``. + Tries GPU params first, then CPU, to support cross-device round-trips. +- Reconstructs the ZFP stream from ``*region_data``. +- Decompresses into a newly ``malloc``'d buffer. +- Restores ``output_region`` to the original pre-compression shape. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_zfp_gpu: + +ZFP Compression — GPU +===================== + +**Source file:** ``pdc_tf_builtin_zfp_gpu.c`` + +**Requires:** ``ENABLE_TF_ZFP_COMPRESSION`` and ``CUDA_ENABLED`` + +Same logical operation as :ref:`tf_builtin_zfp_cpu` but executes on the GPU using +``zfp_stream_set_execution(zfp, zfp_exec_cuda)``. + +pdc_tf_builtin_zfp_compress_cuda +---------------------------------- + +- Copies host input data to device (``cudaMemcpy`` H→D). +- Allocates a device output buffer sized by ``zfp_stream_maximum_size``. +- Sets ZFP CUDA execution mode and calls ``zfp_compress`` on the GPU. +- Synchronizes the device (``cudaDeviceSynchronize``). +- Copies compressed result back to host, frees device buffers. +- Saves params via ``SET_FUNC_PARAMS("zfp_compress", PDC_TF_GPU_DEVICE, ...)``. + +pdc_tf_builtin_zfp_decompress_cuda +------------------------------------ + +- Retrieves saved params (tries GPU first, then CPU). +- Copies compressed data to device. +- Allocates device output buffer of the correct uncompressed size. +- Sets ZFP CUDA execution mode and calls ``zfp_decompress`` on the GPU. +- Validates that decompression returned a non-zero byte count. +- Copies decompressed result back to host. +- Frees all device buffers and ZFP/bitstream resources. + +Error Handling +-------------- + +All CUDA calls are wrapped in ``CUDA_CHECK`` which prints the CUDA error string, +file, and line number, then calls ``exit(EXIT_FAILURE)`` on failure. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_sz_cpu: + +SZ3 Compression — CPU +===================== + +**Source file:** ``pdc_tf_builtin_sz_cpu.c`` + +**Requires:** ``ENABLE_TF_SZ_COMPRESSION`` + +SZ3 is an error-bounded lossy scientific data compressor that achieves high +compression ratios on floating-point arrays by exploiting value predictability. + +Supported Types +--------------- + +``PDC_FLOAT``, ``PDC_DOUBLE``, ``PDC_INT32``, ``PDC_INT64``. +Integer types are mapped to their nearest float equivalent for SZ3 internally +(``PDC_INT32`` → ``SZ_FLOAT``, ``PDC_INT64`` → ``SZ_DOUBLE``). + +Error Bound +----------- + +Uses ``ABS`` mode with ``absErrBound = 0.01``. Each reconstructed value is +guaranteed to differ from the original by no more than 0.01 in absolute terms. + +pdc_tf_builtin_sz_compress +----------------------------- + +- Extracts up to 5 dimension sizes from the input region (unused dimensions default to 1). +- Calls ``SZ_compress_args`` with the absolute error bound configuration. +- Replaces ``*region_data`` with the SZ3-allocated compressed buffer. +- Sets output region to ``{ndim=1, pdc_var_type=PDC_CHAR, size[0]=compressed_bytes}``. +- Saves original region via ``SET_FUNC_PARAMS("sz_compress", PDC_TF_CPU_DEVICE, ...)``. + +pdc_tf_builtin_sz_decompress +------------------------------ + +- Retrieves the saved region descriptor. +- Calls ``SZ_decompress`` with the original dimension sizes. +- Restores ``output_region`` to the original pre-compression shape. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_sz_gpu: + +SZ (cuSZ) Compression — GPU +============================ + +**Source file:** ``pdc_tf_builtin_sz_gpu.c`` + +**Requires:** ``ENABLE_TF_SZ_GPU_COMPRESSSION`` + +cuSZ is the CUDA GPU port of the SZ compressor. It provides GPU-accelerated +error-bounded lossy compression for scientific floating-point data. + +Supported Types +--------------- + +``PDC_FLOAT`` / ``PDC_INT32`` → cuSZ ``F4``. +``PDC_DOUBLE`` / ``PDC_INT64`` → cuSZ ``F8``. +Supports 1D–3D arrays. + +pdc_tf_builtin_sz_compress_cuda +--------------------------------- + +- Allocates a GPU buffer and copies input data to device. +- Creates a cuSZ compressor with ``psz_create_default`` using the input dtype and dimensions. +- Runs ``psz_compress`` with ``Abs`` error mode and tolerance ``0.01``. +- Copies the compressed output back to a host ``malloc``'d buffer. +- Frees device buffers and releases the compressor. +- Sets output region to ``{ndim=1, pdc_var_type=PDC_CHAR, size[0]=compressed_bytes}``. + +pdc_tf_builtin_sz_decompress_cuda +----------------------------------- + +- Reconstructs the cuSZ compressor from the compressed data header via ``psz_create_from_header``. +- Copies compressed data to device. +- Allocates a device output buffer sized for the decompressed data (``float`` assumed). +- Runs ``psz_decompress`` and synchronizes the device. +- Copies the decompressed result back to host. +- Cleans up with ``psz_clear_buffer`` and ``psz_release``. + +.. note:: + + The output buffer is always sized as ``num_elements * sizeof(float)`` + regardless of the original integer type. This is a current limitation of the + cuSZ integration. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_encrypt: + +SecretBox Encryption — CPU +=========================== + +**Source file:** ``pdc_tf_builtin_encrypt_cpu.c`` + +**Requires:** ``ENABLE_TF_SECRET_BOX_ENCRYPTION`` + +Uses ``libsodium``'s ``crypto_secretbox_easy`` (XSalsa20-Poly1305) for authenticated +symmetric encryption. Provides both confidentiality and integrity verification. + +Key and Nonce +------------- + +The key and nonce are module-level zero-initialized arrays: + +.. code-block:: c + + unsigned char key[crypto_secretbox_KEYBYTES] = {0}; + unsigned char nonce[crypto_secretbox_NONCEBYTES] = {0}; + +.. warning:: + + These are currently hardcoded to zero and must be configured before production use. + Using a zero key and nonce provides no real security. + +pdc_tf_builtin_encrypt +------------------------ + +- Allocates a ciphertext buffer of size ``plaintext_len + crypto_secretbox_MACBYTES``. + The extra bytes hold the Poly1305 authentication MAC. +- Calls ``crypto_secretbox_easy`` and replaces ``*region_data`` with the ciphertext. +- Sets output region to ``{ndim=1, pdc_var_type=PDC_CHAR, size[0]=ciphertext_len}``. +- Saves the original region via ``SET_FUNC_PARAMS("secret_box_encrypt", PDC_TF_CPU_DEVICE, ...)``. + +pdc_tf_builtin_decrypt +------------------------ + +- Retrieves the saved original region descriptor. +- Calls ``crypto_secretbox_open_easy`` to verify the MAC and decrypt. +- Returns ``false`` immediately if MAC verification fails — this indicates either + data corruption or tampering. +- On success, restores ``output_region`` to the original unencrypted shape. + + +------------------------------------------------------------------------------- + +.. _tf_builtin_turbo: + +Turbo Compression — CPU +======================== + +**Source file:** ``pdc_tf_builtin_turbo_cpu.c`` + +**Requires:** ``ENABLE_TF_TURBO_COMPRESSION`` + +Uses the TurboPFor library (``ic.h``) which provides extremely fast SIMD-accelerated +lossless compression for integer arrays using the P4NENC family of codecs. + +Supported Types +--------------- + +``PDC_INT32`` and ``PDC_INT64`` only. Floating-point types are not supported by +this codec. + +pdc_tf_builtin_turbo_compress +------------------------------- + +- Calls ``p4nenc32`` for 32-bit integers or ``p4nenc64`` for 64-bit integers. +- The output buffer is pre-allocated at the same size as the input (worst case). +- Sets output region to ``{ndim=1, pdc_var_type=PDC_CHAR, size[0]=compressed_bytes}`` + where ``compressed_bytes`` is the actual encoded size returned by the codec. +- Saves the original region via ``SET_FUNC_PARAMS("turbo_compress", PDC_TF_CPU_DEVICE, ...)``. + +pdc_tf_builtin_turbo_decompress +--------------------------------- + +- Retrieves the saved region descriptor to determine the number of elements and output size. +- Calls ``p4ndec32`` or ``p4ndec64`` to decode into a newly ``malloc``'d buffer. +- Restores ``output_region`` to the original pre-compression shape. + +.. note:: + + TurboPFor codecs are lossless. Unlike SZ3, there is no error bound — the + reconstructed values are bit-for-bit identical to the originals. + + +------------------------------------------------------------------------------- + +.. _tf_external_transforms: + +Adding External Transformations +================================ + +You can extend the framework with your own transformation functions without modifying +PDC itself by writing a shared library and referencing it from a JSON graph file. + +Step 1 — Write the Function +---------------------------- + +Your function must match the ``c_func_t`` prototype exactly: + +.. code-block:: c + + #include "pdc_tf_user.h" + + bool my_transform(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) + { + // ... transform *region_data in place or replace it ... + return true; + } + +Compile it as a shared library:: + + gcc -shared -fPIC -o libmytransform.so my_transform.c + +Step 2 — Add to the JSON Graph +-------------------------------- + +.. code-block:: json + + { + "name": "my_graph", + "lib_path": "/path/to/libmytransform.so", + "states": [ + { "name": "raw" }, + { "name": "my_format" } + ], + "functions": [ + { + "name": "my_transform", + "device": "CPU", + "location": "external", + "input_state": "raw", + "output_state": "my_format" + } + ] + } + +Step 3 — Use It +---------------- + +.. code-block:: c + + PDCtf_init(); + pdcid_t dg = PDCtf_dg_json_create("/path/to/my_graph.json"); + PDCtf_attach_to_obj(dg, obj_id, "raw", "my_format"); + +The framework will ``dlopen`` your library and ``dlsym`` your function at graph-load +time, then register it in the builtin function registry automatically. + +Notes +----- + +- The function symbol name in the library must exactly match the ``"name"`` field in the JSON. +- Only ``pdc_tf_user.h`` is available to external libraries — do not include any other PDC headers. +- If your function needs to persist parameters between compress/decompress calls, use the + ``SET_FUNC_PARAMS`` / ``GET_FUNC_PARAMS`` macros as described in :ref:`tf_parameter_macros`. diff --git a/examples/C_plus_plus_example/multidataset_plugin.h b/examples/C_plus_plus_example/multidataset_plugin.h index 162c53bfb..2a3dd3fd9 100644 --- a/examples/C_plus_plus_example/multidataset_plugin.h +++ b/examples/C_plus_plus_example/multidataset_plugin.h @@ -1,7 +1,7 @@ #ifndef MULTIDATASET_PLUGIN_H #define MULTIDATASET_PLUGIN_H -//#include +// #include #include #include #include diff --git a/examples/llsm/llsm_importer.c b/examples/llsm/llsm_importer.c index 545219a58..c9c75cee2 100644 --- a/examples/llsm/llsm_importer.c +++ b/examples/llsm/llsm_importer.c @@ -75,7 +75,7 @@ import_to_pdc(image_info_t *image_info, csv_cell_t *fileName_cell) // psize_t ndims = 1; // uint64_t offsets[1] = {0}; - // // FIXME: we should support uint64_t. + // FIXME: we should support uint64_t. // uint64_t dims[1] = {image_info->x * image_info->y * image_info->z}; // FIXME: we should change the ndims parameter to psize_t type. diff --git a/examples/read_write_col_perf.c b/examples/read_write_col_perf.c index 7aa15998a..8b036ee7d 100644 --- a/examples/read_write_col_perf.c +++ b/examples/read_write_col_perf.c @@ -31,7 +31,6 @@ #include #include #include "pdc.h" -#include "pdc_analysis.h" #define BUF_LEN 128 diff --git a/examples/read_write_perf.c b/examples/read_write_perf.c index 923fdae1a..34892c5ab 100644 --- a/examples/read_write_perf.c +++ b/examples/read_write_perf.c @@ -31,7 +31,6 @@ #include #include #include "pdc.h" -#include "pdc_analysis.h" #define BUF_LEN 128 diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index c92b2c3b0..81fa4e73f 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -8,17 +8,17 @@ set(LOCAL_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${PDC_SOURCE_DIR}/src/server/include + ${PDC_SOURCE_DIR}/src/server/transform/include ${PDC_SOURCE_DIR}/src/server/pdc_server_region ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_analysis/include ${PDC_SOURCE_DIR}/src/server/dablooms ${PDC_SOURCE_DIR}/src/api/include ${PDC_SOURCE_DIR}/src/api/pdc_obj/include ${PDC_SOURCE_DIR}/src/api/pdc_region/include ${PDC_SOURCE_DIR}/src/api/pdc_query/include - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include + ${PDC_SOURCE_DIR}/src/api/pdc_hist/include ${PDC_SOURCE_DIR}/src/api/profiling/include + ${PDC_SOURCE_DIR}/src/api/pdc_tf/include ${PDC_SOURCE_DIR}/src/utils/include ) @@ -72,26 +72,33 @@ set(PDC_SRCS ${PDC_SOURCE_DIR}/src/api/pdc_obj/pdc_prop.c ${PDC_SOURCE_DIR}/src/api/pdc_obj/pdc_cont.c ${PDC_SOURCE_DIR}/src/api/pdc_obj/pdc_obj.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c + ${PDC_SOURCE_DIR}/src/api/pdc_hist/pdc_hist_pkg.c ${PDC_SOURCE_DIR}/src/api/pdc_obj/pdc_mpi.c ${PDC_SOURCE_DIR}/src/api/pdc_query/pdc_query.c ${PDC_SOURCE_DIR}/src/api/pdc_region/pdc_region.c ${PDC_SOURCE_DIR}/src/api/pdc_region/pdc_region_transfer.c - ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transform.c - ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c + ${PDC_SOURCE_DIR}/src/api/pdc_tf/pdc_tf.c ${PDC_SOURCE_DIR}/src/server/pdc_client_server_common.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_server.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_common.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_poly_sched.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_user.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_zfp_cpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_zfp_gpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_sz_cpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_sz_gpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_encrypt_cpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_builtin_turbo_cpu.c + ${PDC_SOURCE_DIR}/src/server/transform/pdc_tf_profiler.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c ${PDC_SOURCE_DIR}/src/utils/pdc_interface.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ) + ${PDC_SOURCE_DIR}/src/commons/utils/pdc_malloc.c +) - set(PDC_COMMON_INCLUDE_DIRS ${PDC_COMMON_INCLUDE_DIRS}) - # add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/profiling) +set(PDC_COMMON_INCLUDE_DIRS ${PDC_COMMON_INCLUDE_DIRS}) #------------------------------------------------------------------------------ # Libraries @@ -142,7 +149,6 @@ install( #----------------------------------------------------------------------------- set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/include/pdc.h - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include/pdc_analysis.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_cont.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_mpi.h ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_obj.h @@ -153,7 +159,34 @@ set(PDC_HEADERS ${PDC_SOURCE_DIR}/src/api/pdc_obj/include/pdc_prop_pkg.h ${PDC_SOURCE_DIR}/src/api/pdc_query/include/pdc_query.h ${PDC_SOURCE_DIR}/src/api/pdc_region/include/pdc_region.h - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include/pdc_transform.h + ${PDC_SOURCE_DIR}/src/api/pdc_tf/include/pdc_tf.h + ${PDC_SOURCE_DIR}/src/api/include/pdc_client_connect.h + ${PDC_SOURCE_DIR}/src/server/include/pdc_client_server_common.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_private.h + ${PDC_SOURCE_DIR}/src/server/transform/include/pdc_tf_builtin_common.h + ${PDC_SOURCE_DIR}/src/server/transform/include/pdc_tf_common.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_dg.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/pdc_timing.h + ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include/pdc_server_region_transfer_metadata_query.h + ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include/pdc_server_region_transfer.h + ${PDC_SOURCE_DIR}/src/commons/index/dart/include/dart_core.h + ${PDC_SOURCE_DIR}/src/commons/index/dart/include/dart_algo.h + ${PDC_SOURCE_DIR}/src/commons/index/dart/include/dart_math.h + ${PDC_SOURCE_DIR}/src/commons/index/dart/include/dart_utils.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/thpool.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/string_utils.h + ${PDC_SOURCE_DIR}/src/commons/utils/include/query_utils.h + ${PDC_SOURCE_DIR}/src/commons/collections/libhl/include/comparators.h + ${PDC_SOURCE_DIR}/src/commons/collections/libhl/include/linklist.h + ${PDC_SOURCE_DIR}/src/commons/collections/libhl/include/rbtree.h + ${PDC_SOURCE_DIR}/src/commons/collections/libhl/include/atomic_defs.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_set.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_vector.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_compare.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_hash.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_deque.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/art.h + ${PDC_SOURCE_DIR}/src/commons/collections/include/pdc_hash_table.h ${PROJECT_BINARY_DIR}/pdc_config_sys.h ${PROJECT_BINARY_DIR}/pdc_config.h ) diff --git a/src/api/close_server.c b/src/api/close_server.c index 853826980..346f523d3 100644 --- a/src/api/close_server.c +++ b/src/api/close_server.c @@ -53,10 +53,8 @@ main(int argc, char *argv[]) if (PDCclose(pdc) < 0) LOG_ERROR("Failed to close PDC\n"); + LOG_WARNING("total close time = %lf\n", MPI_Wtime() - start); #ifdef ENABLE_MPI - if (!rank) { - LOG_INFO("total close time = %lf\n", MPI_Wtime() - start); - } MPI_Finalize(); #endif diff --git a/src/api/include/pdc.h b/src/api/include/pdc.h index 714120dab..a53b95d0a 100644 --- a/src/api/include/pdc.h +++ b/src/api/include/pdc.h @@ -38,8 +38,7 @@ #include "pdc_obj.h" #include "pdc_region.h" #include "pdc_query.h" -#include "pdc_analysis.h" -#include "pdc_transform.h" +#include "pdc_tf.h" int PDC_timing_report(const char *prefix); diff --git a/src/api/include/pdc_client_connect.h b/src/api/include/pdc_client_connect.h index e8999ab2f..8201e4561 100644 --- a/src/api/include/pdc_client_connect.h +++ b/src/api/include/pdc_client_connect.h @@ -71,18 +71,6 @@ struct _pdc_client_lookup_args { hg_request_t *request; }; -struct _pdc_client_transform_args { - size_t size; - void * data; - void * transform_result; - struct _pdc_region_transform_ftn_info *this_transform; - struct pdc_region_info * region_info; - int type_extent; - int transform_state; - int ret; - hg_bulk_t local_bulk_handle; -}; - struct _pdc_metadata_query_args { pdc_metadata_t *data; }; @@ -227,7 +215,7 @@ perr_t PDC_Client_send_name_recv_id(const char *obj_name, uint64_t cont_id, pdci perr_t PDC_Client_transfer_request(hg_bulk_t *bulk_handle, void *buf, pdcid_t obj_id, uint32_t data_server_id, int obj_ndim, uint64_t *obj_dims, int remote_ndim, uint64_t *remote_offset, uint64_t *remote_size, size_t unit, pdc_access_t access_type, - pdcid_t *metadata_id); + pdcid_t *metadata_id, struct _pdc_obj_info *obj_pointer); int PDC_Client_get_var_type_size(pdc_var_type_t dtype); diff --git a/src/api/pdc.c b/src/api/pdc.c index 642f19fc2..dcf0a17e7 100644 --- a/src/api/pdc.c +++ b/src/api/pdc.c @@ -32,9 +32,9 @@ #include "pdc_cont_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_region_pkg.h" -#include "pdc_analysis_pkg.h" #include "pdc_interface.h" #include "pdc_client_connect.h" +#include "pdc_tf.h" #include "pdc_timing.h" @@ -104,6 +104,8 @@ PDCinit(const char *pdc_name) PGOTO_ERROR(0, "PDC region init error"); if (PDC_transfer_request_init() < 0) PGOTO_ERROR(0, "PDC region transfer init error"); + if (PDCtf_init() < 0) + PGOTO_ERROR(0, "PDC tf init error"); // PDC Client Server connection init if (PDC_Client_init() < 0) @@ -200,13 +202,6 @@ PDCclose(pdcid_t pdcid) if (PDC_region_end() < 0) PGOTO_ERROR(FAIL, "Failed to destroy region"); - if (PDC_iterator_end() < 0) - PGOTO_ERROR(FAIL, "Failed to destroy iterator"); - if (PDC_analysis_end() < 0) - PGOTO_ERROR(FAIL, "Failed to destroy analysis"); - if (PDC_transform_end() < 0) - PGOTO_ERROR(FAIL, "Failed to destroy transform"); - PDC_class_close(pdcid); PDC_class_end(); diff --git a/src/api/pdc_analysis/include/pdc_analysis.h b/src/api/pdc_analysis/include/pdc_analysis.h deleted file mode 100644 index a6f84c538..000000000 --- a/src/api/pdc_analysis/include/pdc_analysis.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_ANALYSIS_SUPPORT_H -#define PDC_ANALYSIS_SUPPORT_H - -#include "pdc_public.h" - -/*********************/ -/* Public Prototypes */ -/*********************/ -/** - * Create a PDC iterator (basic form which internally calls the block version with contig_blocks = 1) - * - * \param obj_id [IN] PDC object ID - * \param reg_id [IN] PDC region ID - * - * \return Iterator id on success/Zero on failure - */ -pdcid_t PDCobj_data_iter_create(pdcid_t obj_id, pdcid_t reg_id); - -/** - * Create a PDC block iterator (general form) - * - * \param obj_id [IN] PDC object ID - * \param reg_id [IN] PDC region ID - * \param contig_blocks [IN] How many rows or columns the iterator should return - * - * \return Iterator id on success/Zero on failure - */ -pdcid_t PDCobj_data_block_iterator_create(pdcid_t obj_id, pdcid_t reg_id, int contig_blocks); - -/** - * ***** - * - * \param iter [IN] The number iteration - * - * \return ******** - */ -size_t PDCobj_data_getSliceCount(pdcid_t iter); - -/** - * Use a PDC data iterator to retrieve object data for PDC in-locus Analysis - * - * \param iter [IN] PDC iterator id - * \param nextBlock [OUT] Pointer to a contiguous block of memory that contains object data - * \param dims [IN/OUT] A pointer to an array (2D) of rows/columns - * - * \return Total number of elements contained by the datablock. - */ -size_t PDCobj_data_getNextBlock(pdcid_t iter, void **nextBlock, size_t *dims); - -/** - * Register a function to be invoked at registration time and/or when data buffers - * referenced by the input iterator is modified. - * - * \param func [IN] String containing the [libraryname:]function to be registered. - * (default library name = "libpdcanalysis") - * \param iterIn [IN] PDC iterator id containing the input data - * (may be a NULL iterator == 0). - * \param dims [IN] PDC iterator id containing the output data - * (may be a NULL iterator == 0). - * - * \return Non-negative on success/Negative on failure - * NOTES: - * In the use case where null iterators are supplied, the function will always be - * invoked on the data server. Because there are no data references, the supplied function - * will only be called at registration time. This can be used to invoke special functions - * such as the reading of data from files (HDF5, etc.) or to start monitoring or timing - * functions. One could for example provide an API to start or stop profilers, take data - * snapshots, etc. - */ -perr_t PDCobj_analysis_register(char *func, pdcid_t iterIn, pdcid_t iterOut); - -/** - * **** - * - * \return **** - */ -int PDCiter_get_nextId(void); - -#endif /* PDC_ANALYSIS_SUPPORT_H */ diff --git a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h b/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h deleted file mode 100644 index d812c65c7..000000000 --- a/src/api/pdc_analysis/include/pdc_analysis_and_transforms_common.h +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ -#ifndef PDC_OBJ_ANALYSIS_H -#define PDC_OBJ_ANALYSIS_H - -#include "pdc_prop_pkg.h" -#include "pdc_transforms_pkg.h" -#include "pdc_timing.h" -#include "mercury_proc_string.h" -#include "mercury_atomic.h" -#include - -#define PDC_REGION_ALL (pdcid_t)(-1) -#define PDC_ITER_NULL (pdcid_t)(0) - -extern int pdc_server_num_g; -extern int pdc_client_mpi_rank_g; -extern struct _pdc_server_info *pdc_server_info_g; - -struct _pdc_region_analysis_ftn_info { - int meta_index; - int n_args; - pdcid_t *object_id; - pdcid_t *region_id; - int (*ftnPtr)(); - int readyCount; - int client_id; - int ftn_lastResult; - _pdc_analysis_language_t lang; - void * data; -}; - -/* - * The basic idea for introducing an iterator data structure is that - * this approach allows "lazy" processing for accessing sub-regions. - * The tradeoff is that by introducing temp arrays to either capture - * an input region or an output region one allows user code - * to have illusion of continguous storage. The price to paid however, - * is that for input data the object content needs to be copied from the - * base data container into the temp storage. On output, the problem - * is somewhat exasperated by the fact that post execution (which may - * be asynchronous), the generated output data needs to be copied back - * into the actual object storage. - * - * Design of these simple iterators is straight forward; we maintain - * the iterator 'state' within the interator_info struct (shown - * below), which we index using the pdcid_t that was returned when the - * iterator was initialized. There's enough info contained within to - * structure allow a reset operation if required. Upon free'ing the struct, - * we enter the free index into a second cache which we access in a - * LIFO manner. Eventually, if a free index cannot be found, we - * reallocate a larger iterator cache, copy the state contained in the old, - * to the new and then fill the next new entry and continue. - * - * Special cases: - * We've added "block" iterators which are a convenient mechanism for - * thread parallelism. The basic idea is that we can farm out some number of - * data blocks to new threads for analysis. This provides a basic control - * mechanism to avoid creating and possibly destroying threads for each - * row or column. - * Expanding on the above capability, when users specify block iterators - * in conjunction with subregions, then we alloc a temporary data buffer - * which is subsequently filled from the specified subregion. - * CAUTION: the copy_region needs to be managed on a per-thread basis! - */ - -struct _pdc_iterator_info { - void * srcStart; /**** Constant that points to the data buffer */ - void * srcNext; /**** Updated data pointer for each iteration */ - void * copy_region; /* Normally unused (see special cases) */ - size_t sliceCount; /**** Total # of slices to return */ - size_t sliceNext; /* Current count that we are going to return */ - size_t sliceResetCount; /* For 3d, when to recalculate 'srcNext' */ - size_t elementsPerSlice; /* # of elements in a slice the data */ - size_t slicePerBlock; /* # of slices to be returned to the user */ - size_t elementsPerPlane; /* rows * columns */ - size_t elementsPerBlock; /* Total elements in a block (return value?) */ - size_t skipCount; /* Offset from the start of a new plane (Used to initialize srcNext) */ - size_t element_size; /* Byte length of a single object element */ - size_t srcBlockCount; /* Current count of 2d blocks */ - size_t contigBlockSize; /* Number of elements in each slice (bytes) (Used to move to the next Block) */ - size_t totalElements; - size_t dims[4]; /* [ planes, rows,columns] */ - size_t *srcDims; /* Values passed into create_iterator */ - size_t ndim; /* number of values in srcDims */ - size_t startOffset; /* Used to initialize the srcNext field */ - pdc_var_type_t pdc_datatype; - _pdc_major_type_t storage_order; /* Copied from the object storage order */ - pdcid_t objectId; /* Reference object ID */ - pdcid_t reg_id; /* Reference region ID */ - pdcid_t local_id; /* Our local reference id */ - pdcid_t meta_id; /* The server registration id */ -}; - -/* struct used to carry state of overall RPC operation across callbacks - * Modified from Mercury: examples - */ -struct _pdc_my_rpc_state { - hg_int64_t value; - hg_size_t size; - void * buffer; - hg_bulk_t bulk_handle; - hg_handle_t handle; -}; - -struct _pdc_iterator_cbs_t { - size_t (*getSliceCount)(pdcid_t); - size_t (*getNextBlock)(pdcid_t, void **, size_t *); -}; - -// Analysis -#define CACHE_SIZE 8192 -extern struct _pdc_iterator_info *PDC_Block_iterator_cache; -extern _pdc_loci_t execution_locus; - -typedef struct analysis_ftn_in_t { - hg_const_string_t ftn_name; - hg_const_string_t loadpath; - pdcid_t local_obj_id; - pdcid_t iter_in; - pdcid_t iter_out; -} analysis_ftn_in_t; - -typedef struct analysis_ftn_out_t { - uint64_t remote_ftn_id; -} analysis_ftn_out_t; - -typedef struct obj_data_iterator_in_t { - pdcid_t client_iter_id; - pdcid_t object_id; - pdcid_t reg_id; - uint64_t sliceCount; - uint64_t sliceNext; - uint64_t sliceResetCount; - uint64_t elementsPerSlice; - uint64_t slicePerBlock; - uint64_t elementsPerPlane; - uint64_t elementsPerBlock; - uint64_t skipCount; - uint64_t element_size; - uint64_t srcBlockCount; - uint64_t contigBlockSize; - uint64_t totalElements; - uint64_t ndim; - uint64_t dims_0; - uint64_t dims_1; - uint64_t dims_2; - uint64_t dims_3; - /* - * The datatype isn't strictly needed but it can be nice - * to have if we eventually provide a default 'fill value'. - * This would be used when the server creates a temp in - * place of a mapped region. - * Generally, we assume that either a region is mapped - * we haven't mapped because the object is an output - * and we really don't care what the initial values are. - * Note we package storage_order and pdc_datatype - * into storage_info... - * +--------+---------------+---------------+ - * |XXXXXXX | storage-order | pdc_datatype | - * +---//---+---------------+---------------+ - * 31 16 15 8 7 0 - */ - int storageinfo; - int server_id; -} obj_data_iterator_in_t; - -typedef struct obj_data_iterator_out_t { - uint64_t server_iter_id; - uint64_t client_iter_id; - /* - * If the container/region utilized by this iterator - * doesn't exist on the server, then it provides a temporary - * region with dimensions provided in the obj_data_iterator_in_t. - * If not temporary is created, then the 'server_region_id' is - * set to -1; - */ - uint64_t server_region_id; - int32_t ret; -} obj_data_iterator_out_t; - -static HG_INLINE hg_return_t -hg_proc_analysis_ftn_in_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - analysis_ftn_in_t *struct_data = (analysis_ftn_in_t *)data; - ret = hg_proc_hg_const_string_t(proc, &struct_data->ftn_name); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_hg_const_string_t(proc, &struct_data->loadpath); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->local_obj_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->iter_in); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->iter_out); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -static HG_INLINE hg_return_t -hg_proc_obj_data_iterator_in_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - obj_data_iterator_in_t *struct_data = (obj_data_iterator_in_t *)data; - ret = hg_proc_uint64_t(proc, &struct_data->client_iter_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->object_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->reg_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->sliceCount); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->sliceNext); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->sliceResetCount); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->elementsPerSlice); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->slicePerBlock); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->elementsPerPlane); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->elementsPerBlock); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->skipCount); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->element_size); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->srcBlockCount); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->contigBlockSize); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->totalElements); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->ndim); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->dims_0); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->dims_1); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->dims_2); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->dims_3); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->storageinfo); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->server_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -static HG_INLINE hg_return_t -hg_proc_analysis_ftn_out_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - analysis_ftn_out_t *struct_data = (analysis_ftn_out_t *)data; - - ret = hg_proc_uint64_t(proc, &struct_data->remote_ftn_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -static HG_INLINE hg_return_t -hg_proc_obj_data_iterator_out_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - obj_data_iterator_out_t *struct_data = (obj_data_iterator_out_t *)data; - - ret = hg_proc_uint64_t(proc, &struct_data->server_iter_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->client_iter_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->server_region_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->ret); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -/***************************************/ -/* Library-private Function Prototypes */ -/***************************************/ -/** - * Registers an iterator - * - * \param iter_id [IN] The ID of iteration - * \param meta_id [OUT] Metadata ID - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id); - -/** - * ******* - * - * \param iterthisFtn_id [IN] ***** - * \param func [IN] ***** - * \param loadpath [IN] ***** - * \param in_local [IN] ***** - * \param out_local [OUT] ***** - * \param in_meta [IN] ***** - * \param out_meta [OUT] ***** - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, const char *func, - const char *loadpath, pdcid_t in_local, pdcid_t out_local, - pdcid_t in_meta, pdcid_t out_meta); - -/** - * ****** - * - * \param func [IN] Name of the function - * \param loadpath [IN] Name of the path - * \param src_region_id [IN] ID of source region - * \param dest_region_id [IN] ID of destination region - * \param obj_id [IN] ID of the object - * \param start_state [IN] Start start - * \param next_state [IN] Next state - * \param op_type [IN] Operation type - * \param when [IN] When to start transformation - * \param client_index [IN] Client index - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_Client_register_region_transform(const char *func, const char *loadpath, - pdcid_t src_region_id ATTRIBUTE(unused), pdcid_t dest_region_id, - pdcid_t obj_id, int start_state, int next_state, int op_type, - int when, int client_index); - -#endif /* PDC_OBJ_ANALYSIS_H */ diff --git a/src/api/pdc_analysis/include/pdc_analysis_pkg.h b/src/api/pdc_analysis/include/pdc_analysis_pkg.h deleted file mode 100644 index a4c888946..000000000 --- a/src/api/pdc_analysis/include/pdc_analysis_pkg.h +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ -#ifndef PDC_ANALYSIS_COMMON_H -#define PDC_ANALYSIS_COMMON_H - -#include "pdc_transforms_pkg.h" -#include "mercury.h" -#include "mercury_proc_string.h" -#include "mercury_atomic.h" - -/*****************************/ -/* Library-private Variables */ -/*****************************/ -hg_id_t analysis_ftn_register_id_g; -hg_id_t transform_ftn_register_id_g; -hg_id_t object_data_iterator_register_id_g; -hg_atomic_int32_t registered_analysis_ftn_count_g; - -extern struct _pdc_region_analysis_ftn_info **pdc_region_analysis_registry; - -/***************************************/ -/* Library-private Function Prototypes */ -/***************************************/ -/** - * ***** - * - * \param registry [IN] ******* - * - * \return ****** - */ -int PDC_get_transforms(struct _pdc_region_transform_ftn_info ***registry); - -/** - * ***** - * - * \param op_type [IN] The type of transformation - * \param dest_region [IN] The struct pointing to destination region - * - * \return ******** - */ -int PDC_check_transform(pdc_obj_transform_t op_type, struct pdc_region_info *dest_region); - -/** - * ******** - * - * \param op_type [IN] The type of transformation - * \param dest_region [IN] The struct pointing to destination region - * - * \return ******** - */ -int PDC_check_analysis(pdc_obj_transform_t op_type, struct pdc_region_info *dest_region); - -/** - * ******** - * - * \param op_type [IN] *********** - * - * \return ******** - */ -int PDC_add_transform_ptr_to_registry_(struct _pdc_region_transform_ftn_info *ftnPtr); - -/** - * ******** - * - * \param client_index [IN] The index of the client - * \param meta_index [IN] ************* - * - * \return - */ -int PDC_update_transform_server_meta_index(int client_index, int meta_index); - -/** - * ******* - * - * \param registry [IN] ************ - * - * \return - */ -int PDC_get_analysis_registry(struct _pdc_region_analysis_ftn_info ***registry); - -/** - * ******* - * - * \param locus_identifier [IN] ********** - */ -void PDC_set_execution_locus(_pdc_loci_t locus_identifier); - -/** - * ***** - * - * \param ftn_infoPtr [IN] ********* - * - * \return ***** - */ -int PDC_add_analysis_ptr_to_registry_(struct _pdc_region_analysis_ftn_info *ftn_infoPtr); - -/** - * ***** - * - * \param ftn [IN] ******* - * \param loadpath [IN] ******* - * \param ftnPtr [IN] ******* - * - * \return ***** - */ -int PDC_get_ftnPtr_(const char *ftn, const char *loadpath, void **ftnPtr); - -/** - * ***** - * - * \return - */ -_pdc_loci_t PDC_get_execution_locus(); - -/** - * ******** - * - * \param workingDir [IN] Path of working directory - * \param application [IN] Name of the application - * - * \return **** - */ -char *PDC_find_in_path(char *workingDir, char *application); - -/** - * ******** - * - * \return **** - */ -char *PDC_get_argv0_(); - -/** - * ******** - * - * \param fname [IN] Path of working directory - * \param app_path [IN] Name of the application - * - * \return **** - */ -char *PDC_get_realpath(char *fname, char *app_path); - -/** - * ******** - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_iterator_end(); - -/** - * ******** - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_analysis_end(); - -/** - * ******** - */ -void PDC_free_analysis_registry(); - -/** - * ******** - */ -void PDC_free_transform_registry(); - -/** - * ******** - */ -void PDC_free_iterator_cache(); - -#endif /* PDC_ANALYSIS_COMMON_H */ diff --git a/src/api/pdc_analysis/pdc_analysis.c b/src/api/pdc_analysis/pdc_analysis.c deleted file mode 100644 index 872c9a403..000000000 --- a/src/api/pdc_analysis/pdc_analysis.c +++ /dev/null @@ -1,648 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include - -#include "mercury_atomic.h" - -#include "pdc_config.h" -#include "pdc_utlist.h" -#include "pdc_obj.h" -#include "pdc_malloc.h" -#include "pdc_prop_pkg.h" -#include "pdc_obj_pkg.h" -#include "pdc_prop.h" -#include "pdc_region.h" -#include "pdc_analysis.h" -#include "pdc_analysis_pkg.h" -#include "pdc_analysis_and_transforms_common.h" -#include "pdc_client_server_common.h" - -static char *default_pdc_analysis_lib = "libpdcanalysis.so"; - -#define UNUSED(x) (void)(x) - -#ifndef IS_PDC_SERVER -void * -PDC_Server_get_region_data_ptr(pdcid_t object_id) -{ - FUNC_ENTER(NULL); - UNUSED(object_id); - FUNC_LEAVE(NULL); -} -#endif - -static int -iterator_init(pdcid_t objectId, pdcid_t reg_id, int blocks, struct _pdc_iterator_info *iter) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - int k, d_offset; - size_t i, element_size = 0; - char * data = NULL; - size_t sliceCount = 1; - size_t elementsPerSlice = 1; - size_t elementsPerBlock; - size_t obj_elementsPerSlice; - size_t startOffset = 0; - size_t skipCount = 0; - size_t totalElements; - struct pdc_region_info *region_info = NULL; - struct _pdc_obj_info * object_info = PDC_obj_get_info(objectId); - struct _pdc_obj_prop * obj_prop_ptr; - - /* Gather Information about the underlying object, e.g. - * the object data type, object size, etc. refers to the - * containing array. These are for the most part, applicable - * by the iterator for managing a potentially smaller region - * (see region size info). - */ - if ((obj_prop_ptr = object_info->obj_pt) != NULL) { - if ((iter->storage_order = obj_prop_ptr->transform_prop.storage_order) == ROW_major) { - obj_elementsPerSlice = obj_prop_ptr->obj_prop_pub->dims[obj_prop_ptr->obj_prop_pub->ndim - 1]; - if (obj_prop_ptr->obj_prop_pub->ndim > 2) { - obj_elementsPerSlice *= - obj_prop_ptr->obj_prop_pub->dims[obj_prop_ptr->obj_prop_pub->ndim - 2]; - } - } - else { - /* Is this ok for Fortran? */ - obj_elementsPerSlice = obj_prop_ptr->obj_prop_pub->dims[0]; - /* - if (obj_prop_ptr->ndim > 1) - obj_slicePerBlock = obj_prop_ptr->dims[1]; */ - if (obj_prop_ptr->obj_prop_pub->ndim > 2) { - obj_elementsPerSlice *= obj_prop_ptr->obj_prop_pub->dims[1]; - } - } - iter->totalElements = 1; - if ((iter->srcDims = (size_t *)PDC_calloc(obj_prop_ptr->obj_prop_pub->ndim, sizeof(size_t))) != - NULL) { - iter->ndim = obj_prop_ptr->obj_prop_pub->ndim; - for (i = 0; i < iter->ndim; i++) { - iter->srcDims[i] = (size_t)obj_prop_ptr->obj_prop_pub->dims[i]; - iter->totalElements *= iter->srcDims[i]; - } - } - - /* Data TYPE and SIZE */ - iter->pdc_datatype = obj_prop_ptr->obj_prop_pub->type; - - if ((element_size = obj_prop_ptr->type_extent) == 0) - element_size = PDC_get_var_type_size(obj_prop_ptr->obj_prop_pub->type); - - /* 'contigBlockSize' is the increment amount to move from - * the current data pointer to the start of the next slice. - */ - iter->contigBlockSize = obj_elementsPerSlice * element_size; - } - else - PGOTO_ERROR(FAIL, "Error: Object (%" PRIu64 ") has not been initalized correctly", objectId); - - iter->element_size = element_size; - iter->objectId = objectId; - iter->reg_id = reg_id; - - if (reg_id == PDC_REGION_ALL) { /* Special handling:: We'll provide entire slices */ - iter->srcStart = data; /* (rows,columns,planes) for each successive call */ - iter->srcNext = data; /* of pdc_getNextBlock. */ - - iter->elementsPerSlice = iter->slicePerBlock /* values are those signifying a single slice */ - = iter->contigBlockSize = obj_elementsPerSlice; - iter->contigBlockSize *= element_size; /* Increment value that is used to increment */ - /* the data pointer to the next row... */ - sliceCount = iter->totalElements / obj_elementsPerSlice; - iter->sliceCount = sliceCount; - iter->sliceResetCount = sliceCount + 1; /* Never */ - } - else { - /* The elementsPerSlice (or row length in bytes for this case), is used - * to calculate how much to advance the current iterator data pointer - * to point to the start of a new row, once we've found the start - * of a particular block. Note that for dimensions greater than 2, - * there are TWO steps that take to get to the start of a new block, i.e. - * 1. Using the block number, we use the object dimensions to find - * the start of a block and then; - * 2. Add skipCount to point to the element within a row of the block. - * - * Subsequent advancement within a 2D block is accomplished - * by adding the byte length of an object ROW, e.g. - * - * +-----------------------+ - * | 1 | 2 | 3 | 4 | Example: we want the interior region - * +-----============------+ 6 7 - * | 5 || 6 | 7 || 8 | 10 11 - * +-----------------------+ 14 15 - * | 9 || 10 | 11 || 12 | - * +-----------------------+ Once the data pointer is pointing at '1' - * | 13 || 14 | 15 || 16 | we add the skipCount of 5 x elementSize - * +-----============------+ and return the datapointer and a - * | 17 | 18 | 19 | 19 | contigBlockSize of 2. - * +-----------------------+ We also add 5 x elementSize to srcNext - * to now point at element '10' - * - */ - - region_info = PDCregion_get_info(reg_id); - - /* How many rows in the selected region? */ - if (iter->storage_order == ROW_major) { - iter->dims[0] = 1; - sliceCount = region_info->size[0]; - totalElements = sliceCount; - for (i = 1; i < region_info->ndim; i++) { - iter->dims[i] = region_info->size[i]; - elementsPerSlice *= region_info->size[i]; - } - totalElements = elementsPerSlice * region_info->size[0]; - } - else { - k = (int)iter->ndim - 1; - iter->dims[k] = 1; - sliceCount = region_info->size[k]; - totalElements = sliceCount; - for (; k >= 0; k--) { - iter->dims[k] = region_info->size[k]; - elementsPerSlice *= region_info->size[k]; - } - totalElements = elementsPerSlice * region_info->size[iter->ndim - 1]; - } - - if (totalElements == iter->totalElements) { - iter->elementsPerBlock = elementsPerSlice; - } - /* For regions, the total element count is that of the region - * not the size of the containing object... - */ - iter->totalElements = totalElements; - if (region_info->ndim > 1) { - skipCount = - ((region_info->offset[1] * obj_prop_ptr->obj_prop_pub->dims[0]) + region_info->offset[0]) * - element_size; - iter->sliceResetCount = iter->slicePerBlock = region_info->size[1] + 1; - elementsPerBlock = - obj_prop_ptr->obj_prop_pub->dims[0] * obj_prop_ptr->obj_prop_pub->dims[1] * element_size; - - if (region_info->ndim > 2) { - d_offset = obj_prop_ptr->obj_prop_pub->dims[0] * obj_prop_ptr->obj_prop_pub->dims[1]; - iter->sliceResetCount = region_info->size[1]; - for (i = 2; i < region_info->ndim; i++) { - if (region_info->offset[i] > 0) { - startOffset = d_offset * region_info->offset[i]; - } - } - startOffset *= element_size; - iter->srcBlockCount = startOffset / elementsPerBlock; - } - } - else { - skipCount = region_info->offset[0] * element_size; - } - /* These next should be filled in at the first call - * to PDC_getNextBlock(). - */ - iter->srcStart = data; - iter->srcNext = data + startOffset + skipCount; - iter->startOffset = startOffset; - iter->sliceCount = sliceCount; - iter->skipCount = skipCount; - } - if (blocks > 1) { - iter->contigBlockSize *= blocks; - iter->slicePerBlock *= blocks; - iter->elementsPerBlock *= blocks; - iter->sliceCount /= blocks; - if (iter->sliceResetCount) - iter->sliceResetCount = iter->sliceCount + 1; - if (iter->storage_order == ROW_major) - iter->dims[0] = blocks; - else - iter->dims[1] = blocks; - } - - ret_value = 0; - -done: - FUNC_LEAVE(ret_value); -} - -pdcid_t -PDCobj_data_block_iterator_create(pdcid_t obj_id, pdcid_t reg_id, int contig_blocks) -{ - FUNC_ENTER(NULL); - - pdcid_t ret_value = 0; - pdcid_t iterId; - struct _pdc_iterator_info *p = NULL; - - iterId = PDCiter_get_nextId(); - - p = &PDC_Block_iterator_cache[iterId]; - if ((iterator_init(obj_id, reg_id, contig_blocks, p)) < 0) - PGOTO_ERROR(0, "PDC iterator_init returned an error"); - - ret_value = p->local_id = iterId; - if (PDC_Client_send_iter_recv_id(iterId, &p->meta_id) != SUCCEED) - PGOTO_ERROR(0, "Unable to register a new iterator"); - -done: - FUNC_LEAVE(ret_value); -} - -/* - * A wrapper to the more general "block_iterator_create" (see above) - */ -pdcid_t -PDCobj_data_iter_create(pdcid_t obj_id, pdcid_t reg_id) -{ - FUNC_ENTER(NULL); - - pdcid_t ret_value = 0; - ret_value = PDCobj_data_block_iterator_create(obj_id, reg_id, 1); - - FUNC_LEAVE(ret_value); -} - -/* - * NOTE: - * Because we use dlopen to dynamically open - * an executable, it may be necessary for the server - * to have the LD_LIBRARY_PATH of the client. - * This can/should be part of the UDF registration - * with the server, i.e. we provide the server - * with: - * a) the full path to the client executable - * which must be compiled with the "-fpie -rdynamic" - * flags. - * b) the contents of the PATH and LD_LIBRARY_PATH - * environment variables. - */ - -char * -PDC_get_argv0_() -{ - FUNC_ENTER(NULL); - - char * ret_value = NULL; - static char *_argv0 = NULL; - char fullPath[PATH_MAX] = { - 0, - }; - char currentDir[PATH_MAX] = { - 0, - }; - pid_t mypid = getpid(); - char * next; - char * procpath = NULL; - FILE * shellcmd = NULL; - size_t cmdLength; - - if (_argv0 == NULL) { - // UNIX derived systems e.g. linux, allow us to find the - // command line as the user (or another application) - // invoked us by reading the /proc/{pid}/cmdline - // file. Note that I'm assuming standard posix - // file paths, so the directory seperator is a foward - // slash ('/') and relative paths will include dot ('.') - // dot-dot (".."). - sprintf(fullPath, "/proc/%u/cmdline", mypid); - procpath = strdup(fullPath); - shellcmd = fopen(procpath, "r"); - if (shellcmd == NULL) { - procpath = (char *)PDC_free(procpath); - PGOTO_ERROR(NULL, "fopen failed"); - } - else { - cmdLength = fread(fullPath, 1, sizeof(fullPath), shellcmd); - if (procpath) - procpath = (char *)PDC_free(procpath); - if (cmdLength > 0) { - _argv0 = strdup(fullPath); - /* truncate the cmdline if any whitespace (space or tab) */ - if ((next = strchr(_argv0, ' ')) != NULL) { - *next = 0; - } - else if ((next = strchr(_argv0, '\t')) != NULL) { - *next = 0; - } - } - fclose(shellcmd); - } - if (_argv0[0] != '/') { - if (getcwd(currentDir, sizeof(currentDir)) == NULL) { - PGOTO_ERROR(NULL, "Very long path name detected."); - } - next = PDC_find_in_path(currentDir, _argv0); - if (next == NULL) - PGOTO_ERROR(NULL, "WARNING: Unable to locate application (%s) in user $PATH", _argv0); - else { - /* Get rid of the copy (strdup) of fullPath now in _argv0. - * and replace it with the next (modified/fully_qualified?) version. - */ - _argv0 = (char *)PDC_free(_argv0); - _argv0 = next; - } - } - } - if (_argv0 == NULL) - PGOTO_ERROR(NULL, "Unable to resolve user application name"); - - ret_value = _argv0; - -done: - FUNC_LEAVE(ret_value); -} - -char * -PDC_get_realpath(char *fname, char *app_path) -{ - FUNC_ENTER(NULL); - - char *ret_value = NULL; - int notreadable; - char fullPath[PATH_MAX] = { - 0, - }; - - do { - if (app_path) { - sprintf(fullPath, "%s/%s", app_path, fname); - app_path = NULL; - } - notreadable = access(fullPath, R_OK); - if (notreadable && (app_path == NULL)) { - perror("access"); - PGOTO_DONE(NULL); - } - } while (notreadable); - - ret_value = strdup(fullPath); - -done: - FUNC_LEAVE(ret_value); -} - -perr_t -PDCobj_analysis_register(char *func, pdcid_t iterIn, pdcid_t iterOut) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; /* Return value */ - void * ftnHandle = NULL; - int (*ftnPtr)(pdcid_t, pdcid_t) = NULL; - struct _pdc_region_analysis_ftn_info *thisFtn = NULL; - struct _pdc_iterator_info * i_in = NULL, *i_out = NULL; - pdcid_t meta_id_in = 0, meta_id_out = 0; - pdcid_t local_id_in = 0, local_id_out = 0; - char * thisApp = NULL; - char * colonsep = NULL; - char * analyislibrary = NULL; - char * applicationDir = NULL; - char * userdefinedftn = NULL; - char * loadpath = NULL; - - thisApp = PDC_get_argv0_(); - if (thisApp) { - applicationDir = dirname(strdup(thisApp)); - } - userdefinedftn = strdup(func); - - if ((colonsep = strrchr(userdefinedftn, ':')) != NULL) { - *colonsep++ = 0; - analyislibrary = colonsep; - } - else - analyislibrary = default_pdc_analysis_lib; - - // TODO: - // Should probably validate the location of the "analysislibrary" - // - loadpath = PDC_get_realpath(analyislibrary, applicationDir); - if (PDC_get_ftnPtr_((const char *)userdefinedftn, (const char *)loadpath, &ftnHandle) < 0) - PGOTO_ERROR(FAIL, "PDC_get_ftnPtr_ returned an error"); - - if ((ftnPtr = ftnHandle) == NULL) - PGOTO_ERROR(FAIL, "Analysis function lookup failed"); - - if ((thisFtn = (struct _pdc_region_analysis_ftn_info *)PDC_malloc( - sizeof(struct _pdc_region_analysis_ftn_info))) == NULL) - PGOTO_ERROR(FAIL, "PDC register_obj_analysis memory allocation failed"); - - thisFtn->ftnPtr = (int (*)())ftnPtr; - thisFtn->n_args = 2; - /* Allocate for iterator ids and region ids */ - if ((thisFtn->object_id = (pdcid_t *)PDC_calloc(4, sizeof(pdcid_t))) != NULL) { - thisFtn->object_id[0] = iterIn; - thisFtn->object_id[1] = iterOut; - } - else - PGOTO_ERROR(FAIL, "PDC register_obj_analysis memory allocation failed - object_ids"); - - thisFtn->region_id = (pdcid_t *)&thisFtn->object_id[2]; - - thisFtn->lang = C_lang; - - if (PDC_Block_iterator_cache) { - if (iterIn != 0) { - i_in = &PDC_Block_iterator_cache[iterIn]; - meta_id_in = i_in->meta_id; - local_id_in = i_in->local_id; - } - if (iterOut != 0) { - i_out = &PDC_Block_iterator_cache[iterOut]; - meta_id_out = i_out->meta_id; - local_id_out = i_out->local_id; - } - } - - PDC_Client_register_obj_analysis(thisFtn, userdefinedftn, loadpath, local_id_in, local_id_out, meta_id_in, - meta_id_out); - - // Add region IDs - thisFtn->region_id[0] = i_in->reg_id; - thisFtn->region_id[1] = i_out->reg_id; - - // Add to our own list of analysis functions - if (PDC_add_analysis_ptr_to_registry_(thisFtn) < 0) - PGOTO_ERROR(FAIL, "PDC unable to register analysis function"); - -done: - if (applicationDir) - applicationDir = (char *)PDC_free(applicationDir); - if (userdefinedftn) - userdefinedftn = (char *)PDC_free(userdefinedftn); - if (loadpath) - loadpath = (char *)PDC_free(loadpath); - - FUNC_LEAVE(ret_value); -} - -size_t -PDCobj_data_getSliceCount(pdcid_t iter) -{ - FUNC_ENTER(NULL); - - size_t ret_value = 0; - struct _pdc_iterator_info *thisIter = NULL; - - /* Special case to handle a NULL iterator */ - if (iter == 0) - PGOTO_DONE(0); - /* FIXME: Should add another check to see that the input - * iter id is in the range of cached values... - */ - if ((PDC_Block_iterator_cache != NULL) && (iter > 0)) { - thisIter = &PDC_Block_iterator_cache[iter]; - PGOTO_DONE(thisIter->sliceCount); - } - -done: - FUNC_LEAVE(ret_value); -} - -size_t -PDCobj_data_getNextBlock(pdcid_t iter, void **nextBlock, size_t *dims) -{ - FUNC_ENTER(NULL); - - size_t ret_value = 0; - struct _pdc_iterator_info *thisIter = NULL; - size_t current_total, remaining, offset; - struct _pdc_obj_info * object_info; - struct _pdc_obj_prop * obj_prop_ptr; - - /* Special case to handle a NULL iterator */ - if (iter == 0) { - if (nextBlock != NULL) - *nextBlock = NULL; - if (dims != NULL) - *dims = 0; - PGOTO_DONE(0); - } - - if ((PDC_Block_iterator_cache != NULL) && (iter > 0)) { - thisIter = &PDC_Block_iterator_cache[iter]; - if (thisIter->srcStart == NULL) { - if (execution_locus == SERVER_MEMORY) { - if ((thisIter->srcNext = PDC_Server_get_region_data_ptr(thisIter->objectId)) == NULL) - thisIter->srcNext = PDC_malloc(thisIter->totalElements * thisIter->element_size); - if ((thisIter->srcStart = thisIter->srcNext) == NULL) - PGOTO_ERROR(0, "Unable to allocate iterator storage"); - - thisIter->srcNext += thisIter->startOffset + thisIter->skipCount; - } - else if (execution_locus == CLIENT_MEMORY) { - object_info = PDC_obj_get_info(thisIter->objectId); - obj_prop_ptr = object_info->obj_pt; - if (obj_prop_ptr) { - thisIter->srcNext = thisIter->srcStart = obj_prop_ptr->buf; - thisIter->srcNext += thisIter->startOffset + thisIter->skipCount; - } - } - } - if (thisIter->srcNext != NULL) { - if (thisIter->sliceNext == thisIter->sliceCount) { - /* May need to adjust the elements in this last - * block... - */ - current_total = thisIter->sliceCount * thisIter->elementsPerBlock; - remaining = 0; - - if (current_total == thisIter->totalElements) { - if (nextBlock) - *nextBlock = NULL; - thisIter->sliceNext = 0; - thisIter->srcNext = NULL; - if (dims) - dims[0] = dims[1] = 0; - if (nextBlock) - *nextBlock = NULL; - PGOTO_DONE(0); - } - if (nextBlock) - *nextBlock = thisIter->srcNext; - thisIter->srcNext = NULL; - remaining = thisIter->totalElements - current_total; - if (dims) { - if (thisIter->storage_order == ROW_major) - dims[0] = remaining / thisIter->elementsPerSlice; - else - dims[1] = remaining / thisIter->elementsPerSlice; - } - PGOTO_DONE(remaining); - } - else if (thisIter->sliceNext && (thisIter->sliceNext % thisIter->sliceResetCount) == 0) { - offset = ++thisIter->srcBlockCount * thisIter->elementsPerBlock; - thisIter->srcNext = thisIter->srcStart + offset + thisIter->skipCount; - if (nextBlock) - *nextBlock = thisIter->srcNext; - } - else { - *nextBlock = thisIter->srcNext; - thisIter->srcNext += thisIter->contigBlockSize; - } - thisIter->sliceNext += 1; - if (dims != NULL) { - dims[0] = thisIter->dims[0]; - if (thisIter->ndim > 1) - dims[1] = thisIter->dims[1]; - if (thisIter->ndim > 2) - dims[2] = thisIter->dims[2]; - if (thisIter->ndim > 3) - dims[2] = thisIter->dims[3]; - } - PGOTO_DONE(thisIter->elementsPerBlock); - } - } - -done: - FUNC_LEAVE(ret_value); -} - -perr_t -PDC_analysis_end() -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - PDC_free_analysis_registry(); - - FUNC_LEAVE(ret_value); -} - -perr_t -PDC_iterator_end() -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - PDC_free_iterator_cache(); - - FUNC_LEAVE(ret_value); -} diff --git a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c b/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c deleted file mode 100644 index 8346f8502..000000000 --- a/src/api/pdc_analysis/pdc_analysis_and_transforms_connect.c +++ /dev/null @@ -1,370 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include "pdc_obj_pkg.h" -#include "pdc_client_connect.h" -#include "pdc_analysis_pkg.h" -#include "pdc_transforms_common.h" - -static hg_context_t * send_context_g = NULL; -static hg_atomic_int32_t *atomic_work_todo_g = NULL; - -/* Forward References:: */ -// Analysis and Transformations - -hg_id_t analysis_ftn_register_id_g; -hg_id_t transform_ftn_register_id_g; -hg_id_t server_transform_ftn_register_id_g; -hg_id_t object_data_iterator_register_id_g; - -static hg_return_t client_register_iterator_rpc_cb(const struct hg_cb_info *info); -static hg_return_t client_register_analysis_rpc_cb(const struct hg_cb_info *info); -static hg_return_t client_register_transform_rpc_cb(const struct hg_cb_info *info); - -/* Client APIs */ - -perr_t -PDC_Client_send_iter_recv_id(pdcid_t iter_id, pdcid_t *meta_id) -{ - FUNC_ENTER(NULL); - - uint64_t ret_value = SUCCEED; - struct _pdc_iterator_info *thisIter = NULL; - struct _pdc_my_rpc_state * my_rpc_state_p; - obj_data_iterator_in_t in; - hg_return_t hg_ret; - int server_id = 0; - int n_retry = 0; - struct _pdc_obj_info * object_info; - - if (atomic_work_todo_g == NULL) { - hg_atomic_init32(atomic_work_todo_g, 0); - } - - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_calloc(1, sizeof(struct _pdc_my_rpc_state)); - if (my_rpc_state_p == NULL) - PGOTO_ERROR(FAIL, "PDC_Client_send_iter_recv_id(): Could not allocate my_rpc_state"); - - if ((PDC_Block_iterator_cache != NULL) && (iter_id > 0)) { - thisIter = &PDC_Block_iterator_cache[iter_id]; - /* Find the server association to the specific object */ - in.client_iter_id = iter_id; - object_info = PDC_obj_get_info(thisIter->objectId); - - /* Co-locate iterator info with the actual object */ - if (object_info != NULL) { - server_id = object_info->obj_info_pub->server_id; - in.object_id = object_info->obj_info_pub->meta_id; - } - else - in.object_id = thisIter->objectId; /* Is this ok? */ - in.reg_id = thisIter->reg_id; - in.sliceCount = thisIter->sliceCount; - in.sliceNext = thisIter->sliceNext; - in.sliceResetCount = thisIter->sliceResetCount; - in.elementsPerSlice = thisIter->elementsPerSlice; - in.slicePerBlock = thisIter->slicePerBlock; - in.elementsPerPlane = thisIter->elementsPerPlane; - in.elementsPerBlock = thisIter->elementsPerBlock; - in.skipCount = thisIter->skipCount; - in.element_size = thisIter->element_size; - in.srcBlockCount = thisIter->srcBlockCount; - in.contigBlockSize = thisIter->contigBlockSize; - in.totalElements = thisIter->totalElements; - in.ndim = thisIter->ndim; - in.dims_0 = thisIter->dims[0]; - in.dims_1 = thisIter->dims[1]; - in.dims_2 = thisIter->dims[2]; - in.dims_3 = thisIter->dims[3]; - in.storageinfo = (int)((thisIter->storage_order << 8) | thisIter->pdc_datatype); - } - - while (pdc_server_info_g[server_id].addr_valid != 1) { - if (n_retry > 0) - break; - if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) - PGOTO_ERROR(FAIL, "Error with PDC_Client_lookup_server"); - - n_retry++; - } - - in.server_id = server_id; - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, object_data_iterator_register_id_g, - &my_rpc_state_p->handle); - hg_ret = HG_Forward(my_rpc_state_p->handle, client_register_iterator_rpc_cb, my_rpc_state_p, &in); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_client_send_iter_recv_id(): Could not start HG_Forward()"); - - hg_atomic_incr32(atomic_work_todo_g); - PDC_Client_check_response(&send_context_g); - - if (my_rpc_state_p->value == 0) { - *meta_id = 0; - PGOTO_DONE(FAIL); - } - - *meta_id = my_rpc_state_p->value; - -done: - HG_Destroy(my_rpc_state_p->handle); - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_free(my_rpc_state_p); - - FUNC_LEAVE(ret_value); -} - -// Callback function for HG_Forward() -// Gets executed after a call to HG_Trigger and the RPC has completed -static hg_return_t -client_register_iterator_rpc_cb(const struct hg_cb_info *info) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - struct _pdc_my_rpc_state *my_rpc_state_p = info->arg; - obj_data_iterator_out_t output; - - if (info->ret == HG_SUCCESS) { - ret_value = HG_Get_output(info->info.forward.handle, &output); - if (ret_value != HG_SUCCESS) - PGOTO_ERROR(FAIL, "Unable to read the server return values"); - - my_rpc_state_p->value = output.server_iter_id; - } - -done: - hg_atomic_decr32(atomic_work_todo_g); - HG_Free_output(info->info.forward.handle, &output); - - FUNC_LEAVE(ret_value); -} - -perr_t -PDC_Client_register_obj_analysis(struct _pdc_region_analysis_ftn_info *thisFtn, const char *func, - const char *loadpath, pdcid_t in_local, pdcid_t out_local, pdcid_t in_meta, - pdcid_t out_meta) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - uint32_t server_id = 0; - hg_return_t hg_ret; - analysis_ftn_in_t in; - struct _pdc_my_rpc_state * my_rpc_state_p; - struct _pdc_iterator_info *thisIter; - int n_retry = 0; - int input_server = -1, output_server = -1; - struct _pdc_obj_info * obj_prop; - - if (atomic_work_todo_g == NULL) { - hg_atomic_init32(atomic_work_todo_g, 0); - } - - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_calloc(1, sizeof(struct _pdc_my_rpc_state)); - if (my_rpc_state_p == NULL) - PGOTO_ERROR(FAIL, "PDC_Client_register_obj_analysis(): Could not allocate my_rpc_state"); - - if (pdc_server_selection_g != PDC_SERVER_DEFAULT) { - thisIter = &PDC_Block_iterator_cache[in_local]; - obj_prop = PDC_obj_get_info(thisIter->objectId); - input_server = obj_prop->obj_info_pub->server_id; - output_server = input_server; - } - else { - if (in_local > 0) { - thisIter = &PDC_Block_iterator_cache[in_local]; - input_server = PDC_get_server_by_obj_id(thisIter->objectId, pdc_server_num_g); - } - if (out_local > 0) { - thisIter = &PDC_Block_iterator_cache[out_local]; - output_server = PDC_get_server_by_obj_id(thisIter->objectId, pdc_server_num_g); - } - } - - if (input_server < 0) { - if (output_server < 0) { - server_id = pdc_client_mpi_rank_g % pdc_server_num_g; - } - else - server_id = output_server; - } - else { - server_id = input_server; - } - - while (pdc_server_info_g[server_id].addr_valid != 1) { - if (n_retry > 0) - break; - if (PDC_Client_lookup_server(server_id, 0) != SUCCEED) - PGOTO_ERROR(FAIL, "Error with PDC_Client_lookup_server"); - - n_retry++; - } - - memset(&in, 0, sizeof(in)); - in.ftn_name = func; - in.loadpath = loadpath; - in.iter_in = in_meta; - in.iter_out = out_meta; - - // We have already filled in the pdc_server_info_g[server_id].addr in previous - // client_test_connect_lookup_cb - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, analysis_ftn_register_id_g, - &my_rpc_state_p->handle); - hg_ret = HG_Forward(my_rpc_state_p->handle, client_register_analysis_rpc_cb, my_rpc_state_p, &in); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "PDC_Client_register_obj_analysis(): Could not start HG_Forward()"); - - hg_atomic_incr32(atomic_work_todo_g); - PDC_Client_check_response(&send_context_g); - - if (my_rpc_state_p->value < 0) { - PGOTO_DONE(FAIL); - } - // Here, we should update the local analysis registry with the returned valued from my_rpc_state_p; - thisFtn->meta_index = my_rpc_state_p->value; - -done: - HG_Destroy(my_rpc_state_p->handle); - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_free(my_rpc_state_p); - - FUNC_LEAVE(ret_value); -} - -// Callback function for HG_Forward() -// Gets executed after a call to HG_Trigger and the RPC has completed -static hg_return_t -client_register_analysis_rpc_cb(const struct hg_cb_info *info) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - struct _pdc_my_rpc_state *my_rpc_state_p = info->arg; - analysis_ftn_out_t output; - - if (info->ret == HG_SUCCESS) { - ret_value = HG_Get_output(info->info.forward.handle, &output); - if (ret_value != HG_SUCCESS) - PGOTO_ERROR(ret_value, "Unable to read the server return values"); - - my_rpc_state_p->value = output.remote_ftn_id; - } - -done: - hg_atomic_decr32(atomic_work_todo_g); - HG_Free_output(info->info.forward.handle, &output); - - FUNC_LEAVE(ret_value); -} - -perr_t -PDC_Client_register_region_transform(const char *func, const char *loadpath, - pdcid_t src_region_id ATTRIBUTE(unused), pdcid_t dest_region_id, - pdcid_t obj_id, int start_state, int next_state, int op_type, int when, - int client_index) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - uint32_t server_id = 0; - hg_return_t hg_ret; - transform_ftn_in_t in; - struct _pdc_obj_info * object_info = NULL; - struct _pdc_my_rpc_state *my_rpc_state_p; - - if (atomic_work_todo_g == NULL) { - hg_atomic_init32(atomic_work_todo_g, 0); - } - - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_calloc(1, sizeof(struct _pdc_my_rpc_state)); - if (my_rpc_state_p == NULL) - PGOTO_ERROR(FAIL, "Could not allocate my_rpc_state"); - - /* Find the server associated with the input object */ - server_id = PDC_get_server_by_obj_id(obj_id, pdc_server_num_g); - object_info = PDC_obj_get_info(obj_id); - memset(&in, 0, sizeof(in)); - in.ftn_name = func; - in.loadpath = loadpath; - if (object_info != NULL) - in.object_id = object_info->obj_info_pub->meta_id; - else - in.object_id = obj_id; - in.region_id = dest_region_id; - - in.operation_type = when; - in.start_state = start_state; - in.next_state = next_state; - in.op_type = op_type & 0xFF; - in.when = when & 0xFF; - in.client_index = client_index; - - // We have already filled in the pdc_server_info_g[server_id].addr in previous - // client_test_connect_lookup_cb - HG_Create(send_context_g, pdc_server_info_g[server_id].addr, transform_ftn_register_id_g, - &my_rpc_state_p->handle); - hg_ret = HG_Forward(my_rpc_state_p->handle, client_register_transform_rpc_cb, my_rpc_state_p, &in); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(FAIL, "Could not start HG_Forward()"); - - hg_atomic_incr32(atomic_work_todo_g); - PDC_Client_check_response(&send_context_g); - - if (my_rpc_state_p->value < 0) { - PGOTO_DONE(FAIL); - } - // Here, we should update the local registry with the returned valued from my_rpc_state_p; - -done: - if (object_info) - PDC_free_obj_info(object_info); - HG_Destroy(my_rpc_state_p->handle); - my_rpc_state_p = (struct _pdc_my_rpc_state *)PDC_free(my_rpc_state_p); - - FUNC_LEAVE(ret_value); -} - -// Callback function for HG_Forward() -// Gets executed after a call to HG_Trigger and the RPC has completed -static hg_return_t -client_register_transform_rpc_cb(const struct hg_cb_info *info) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - transform_ftn_out_t output; - - if (info->ret == HG_SUCCESS) { - ret_value = HG_Get_output(info->info.forward.handle, &output); - if (ret_value != HG_SUCCESS) - PGOTO_ERROR(ret_value, "Unable to read the server return values"); - - PDC_update_transform_server_meta_index(output.client_index, output.ret); - } - -done: - hg_atomic_decr32(atomic_work_todo_g); - HG_Free_output(info->info.forward.handle, &output); - - FUNC_LEAVE(ret_value); -} diff --git a/src/api/pdc_analysis/pdc_analysis_common.c b/src/api/pdc_analysis/pdc_analysis_common.c deleted file mode 100644 index c04b3f7d5..000000000 --- a/src/api/pdc_analysis/pdc_analysis_common.c +++ /dev/null @@ -1,635 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -/************************************************************************ - * This file includes the functionality to support PDC in-locus analysis - ************************************************************************ */ - -#include "pdc_config.h" -#include "mercury.h" -#include "pdc_id_pkg.h" -#include "pdc_obj_pkg.h" -#include "pdc_transforms_pkg.h" -#include "pdc_analysis_and_transforms_common.h" -#include "pdc_client_server_common.h" -#include "pdc_analysis_pkg.h" -#include "pdc_malloc.h" -#include "pdc_region.h" -#include "pdc_server_analysis.h" -#include "pdc_logger.h" - -size_t analysis_registry_size = 0; -size_t transform_registry_size = 0; -hg_atomic_int32_t registered_transform_ftn_count_g; -int * i_cache_freed = NULL; -size_t iterator_cache_entries = CACHE_SIZE; -hg_atomic_int32_t i_cache_index; -hg_atomic_int32_t i_free_index; -_pdc_loci_t execution_locus = UNKNOWN; - -#ifdef ENABLE_MULTITHREAD -extern hg_thread_pool_t *hg_test_thread_pool_g; -extern hg_thread_pool_t *hg_test_thread_pool_fs_g; -#endif - -static inline int -compare_gt(int *a, int b) -{ - return (*a) > (b); -} -struct _pdc_region_analysis_ftn_info ** pdc_region_analysis_registry = NULL; -struct _pdc_region_transform_ftn_info **pdc_region_transform_registry = NULL; - -#ifndef IS_PDC_SERVER -// Dummy function for client to compile, real function is used only by server and code is in pdc_server.c -perr_t -PDC_Server_instantiate_data_iterator(obj_data_iterator_in_t *in ATTRIBUTE(unused), - obj_data_iterator_out_t *out ATTRIBUTE(unused)) -{ - FUNC_ENTER(NULL); - FUNC_LEAVE(SUCCEED); -} -void * -PDC_Server_get_ftn_reference(char *ftn ATTRIBUTE(unused)) -{ - FUNC_ENTER(NULL); - FUNC_LEAVE(NULL); -} -int -PDC_get_analysis_registry(struct _pdc_region_analysis_ftn_info ***registry ATTRIBUTE(unused)) -{ - FUNC_ENTER(NULL); - FUNC_LEAVE(0); -}; -#endif - -/* Internal support functions */ -static int -pdc_analysis_registry_init_(size_t newSize) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - struct _pdc_region_analysis_ftn_info **new_registry; - - if (pdc_region_analysis_registry == NULL) { - new_registry = (struct _pdc_region_analysis_ftn_info **)PDC_calloc(sizeof(void *), newSize); - if (new_registry) { - hg_atomic_init32(®istered_analysis_ftn_count_g, 0); - pdc_region_analysis_registry = new_registry; - analysis_registry_size = newSize; - PGOTO_DONE(newSize); - } - } - else if (newSize > analysis_registry_size) { - new_registry = (struct _pdc_region_analysis_ftn_info **)PDC_calloc(sizeof(void *), newSize); - if (new_registry) { - size_t copysize = analysis_registry_size * sizeof(void *); - memcpy(new_registry, pdc_region_analysis_registry, copysize); - pdc_region_analysis_registry = - (struct _pdc_region_analysis_ftn_info **)PDC_free(pdc_region_analysis_registry); - pdc_region_analysis_registry = new_registry; - analysis_registry_size = newSize; - PGOTO_DONE(newSize); - } - } - -done: - FUNC_LEAVE(ret_value); -} - -static int -pdc_transform_registry_init_(size_t newSize) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - struct _pdc_region_transform_ftn_info **new_registry; - size_t copysize; - - if (pdc_region_transform_registry == NULL) { - new_registry = (struct _pdc_region_transform_ftn_info **)PDC_calloc(sizeof(void *), newSize); - if (new_registry) { - hg_atomic_init32(®istered_transform_ftn_count_g, 0); - pdc_region_transform_registry = new_registry; - transform_registry_size = newSize; - PGOTO_DONE(newSize); - } - } - else if (newSize > transform_registry_size) { - new_registry = (struct _pdc_region_transform_ftn_info **)PDC_calloc(sizeof(void *), newSize); - if (new_registry) { - copysize = transform_registry_size * sizeof(void *); - memcpy(new_registry, pdc_region_transform_registry, copysize); - pdc_region_transform_registry = - (struct _pdc_region_transform_ftn_info **)PDC_free(pdc_region_transform_registry); - pdc_region_transform_registry = new_registry; - transform_registry_size = newSize; - PGOTO_DONE(newSize); - } - } - -done: - FUNC_LEAVE(ret_value); -} - -int -pdc_analysis_registry_finalize_() -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - hg_atomic_int32_t i; - - if ((pdc_region_analysis_registry != NULL) && (analysis_registry_size > 0)) { - while (hg_atomic_get32(&i) > 0) { - if (pdc_region_analysis_registry[i - 1]) - pdc_region_analysis_registry[i - 1] = - (struct _pdc_region_analysis_ftn_info *)PDC_free(pdc_region_analysis_registry[i - 1]); - hg_atomic_decr32(&i); - } - pdc_region_analysis_registry = - (struct _pdc_region_analysis_ftn_info **)PDC_free(pdc_region_analysis_registry); - analysis_registry_size = 0; - hg_atomic_init32(®istered_analysis_ftn_count_g, 0); - } - - FUNC_LEAVE(ret_value); -} - -int -check_analysis(pdc_obj_transform_t op_type ATTRIBUTE(unused), struct pdc_region_info *dest_region) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - int i, count; - - if (analysis_registry_size > 0) { - count = hg_atomic_get32(®istered_analysis_ftn_count_g); - for (i = 0; i < count; i++) { - if (pdc_region_analysis_registry[i]->region_id[0] == dest_region->local_id) { - dest_region->registered_op |= PDC_ANALYSIS; - PGOTO_DONE(1); - } - } - } - -done: - FUNC_LEAVE(ret_value); -} - -int -PDC_add_analysis_ptr_to_registry_(struct _pdc_region_analysis_ftn_info *ftn_infoPtr) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - size_t initial_registry_size = 64; - size_t i, currentCount; - int registry_index; - - if (analysis_registry_size == 0) { - if (pdc_analysis_registry_init_(initial_registry_size) == 0) { - perror("Unable to initialize analysis registry"); - PGOTO_DONE(-1); - } - } - currentCount = (size_t)hg_atomic_get32(®istered_analysis_ftn_count_g); - if (currentCount == analysis_registry_size) { - if (pdc_analysis_registry_init_(analysis_registry_size * 2) == 0) { - perror("memory allocation failed"); - PGOTO_DONE(-1); - } - } - /* If the new function is already registered - * simply return the OLD index. - */ - for (i = 0; i < currentCount; i++) { - if (ftn_infoPtr->ftnPtr == pdc_region_analysis_registry[i]->ftnPtr) { - PGOTO_DONE((int)i); /* Found match */ - } - } - registry_index = hg_atomic_get32(®istered_analysis_ftn_count_g); - hg_atomic_incr32(®istered_analysis_ftn_count_g); - pdc_region_analysis_registry[registry_index] = ftn_infoPtr; - - ret_value = registry_index; - -done: - FUNC_LEAVE(ret_value); -} - -int -PDCiter_get_nextId(void) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - size_t nextId = 0; - int * previous_i_cache_freed = 0; - int next_free = 0; - struct _pdc_iterator_info *previous_state; - - if (PDC_Block_iterator_cache == NULL) { - PDC_Block_iterator_cache = (struct _pdc_iterator_info *)PDC_calloc(iterator_cache_entries, - sizeof(struct _pdc_iterator_info)); - if (PDC_Block_iterator_cache == NULL) - PGOTO_ERROR(FAIL, "calloc failed"); - - i_cache_freed = (int *)PDC_calloc(iterator_cache_entries, sizeof(int)); - /* Index 0 is NOT-USED other than to indicate an empty iterator */ - hg_atomic_init32(&i_cache_index, 1); - hg_atomic_init32(&i_free_index, 0); - } - - if (hg_atomic_get32(&i_free_index) > 0) { - next_free = hg_atomic_decr32(&i_free_index); - nextId = i_cache_freed[next_free]; - } - else { - next_free = hg_atomic_incr32(&i_cache_index); - nextId = next_free - 1; /* use the "current" index */ - } - - if (nextId == iterator_cache_entries) { - /* Realloc the cache and free list */ - previous_i_cache_freed = i_cache_freed; - previous_state = PDC_Block_iterator_cache; - PDC_Block_iterator_cache = (struct _pdc_iterator_info *)PDC_calloc(iterator_cache_entries * 2, - sizeof(struct _pdc_iterator_info)); - memcpy(PDC_Block_iterator_cache, previous_state, - iterator_cache_entries * sizeof(struct _pdc_iterator_info)); - i_cache_freed = (int *)PDC_calloc(iterator_cache_entries * 2, sizeof(int)); - memcpy(i_cache_freed, previous_i_cache_freed, iterator_cache_entries * sizeof(int)); - iterator_cache_entries *= 2; - previous_i_cache_freed = (int *)PDC_free(previous_i_cache_freed); - previous_state = (struct _pdc_iterator_info *)PDC_free(previous_state); - } - - ret_value = nextId; - -done: - FUNC_LEAVE(ret_value); -} - -/* - * Analysis and Transform - */ -int -PDC_check_transform(pdc_obj_transform_t op_type, struct pdc_region_info *dest_region) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - int i, count; - - if (transform_registry_size > 0) { - count = hg_atomic_get32(®istered_transform_ftn_count_g); - for (i = 0; i < count; i++) { - if ((pdc_region_transform_registry[i]->op_type == op_type) && - (pdc_region_transform_registry[i]->dest_region == dest_region)) { - dest_region->registered_op |= PDC_TRANSFORM; - PGOTO_DONE(1); - } - } - } - -done: - FUNC_LEAVE(ret_value); -} - -int -PDC_get_transforms(struct _pdc_region_transform_ftn_info ***registry) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - - if (registry) { - *registry = pdc_region_transform_registry; - PGOTO_DONE(hg_atomic_get32(®istered_transform_ftn_count_g)); - } - -done: - FUNC_LEAVE(ret_value); -} - -int -PDC_add_transform_ptr_to_registry_(struct _pdc_region_transform_ftn_info *ftn_infoPtr) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - size_t initial_registry_size = 64; - size_t i, currentCount; - int registry_index; - - if (transform_registry_size == 0) { - if (pdc_transform_registry_init_(initial_registry_size) == 0) - PGOTO_ERROR(FAIL, "Unable to initialize transform registry"); - } - currentCount = (size_t)hg_atomic_get32(®istered_transform_ftn_count_g); - if (currentCount == transform_registry_size) { - if (pdc_transform_registry_init_(transform_registry_size * 2) == 0) - PGOTO_ERROR(FAIL, "Memory allocation failed"); - } - /* If the new function is already registered - * simply return the OLD index. - */ - for (i = 0; i < currentCount; i++) { - if ((ftn_infoPtr->ftnPtr == pdc_region_transform_registry[i]->ftnPtr) && - (ftn_infoPtr->object_id == pdc_region_transform_registry[i]->object_id)) { - PGOTO_DONE((int)i); /* Found match */ - } - } - - registry_index = hg_atomic_get32(®istered_transform_ftn_count_g); - hg_atomic_incr32(®istered_transform_ftn_count_g); - pdc_region_transform_registry[registry_index] = ftn_infoPtr; - - ret_value = registry_index; - -done: - FUNC_LEAVE(ret_value); -} - -int -PDC_update_transform_server_meta_index(int client_index, int meta_index) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - struct _pdc_region_transform_ftn_info *ftnPtr; - - if (client_index < registered_transform_ftn_count_g) { - ftnPtr = pdc_region_transform_registry[client_index]; - ftnPtr->meta_index = meta_index; - } - else - PGOTO_ERROR(FAIL, "Bad client index(%d)", client_index); - -done: - FUNC_LEAVE(ret_value); -} - -void -PDC_set_execution_locus(_pdc_loci_t locus_identifier) -{ - FUNC_ENTER(NULL); - - execution_locus = locus_identifier; - - FUNC_LEAVE_VOID(); -} - -_pdc_loci_t -PDC_get_execution_locus() -{ - FUNC_ENTER(NULL); - - _pdc_loci_t ret_value = 0; - ret_value = execution_locus; - - FUNC_LEAVE(ret_value); -} - -int -PDC_get_ftnPtr_(const char *ftn, const char *loadpath, void **ftnPtr) -{ - FUNC_ENTER(NULL); - - int ret_value = 0; - static void *appHandle = NULL; - void * ftnHandle = NULL; - char * this_error = NULL; - - if (appHandle == NULL) { - if ((appHandle = dlopen(loadpath, RTLD_NOW)) == NULL) { - this_error = dlerror(); - PGOTO_ERROR(FAIL, "dlopen failed: %s", this_error); - } - } - ftnHandle = dlsym(appHandle, ftn); - if (ftnHandle == NULL) - PGOTO_ERROR(FAIL, "dlsym failed: %s", dlerror()); - - *ftnPtr = ftnHandle; - - ret_value = 0; - -done: - FUNC_LEAVE(ret_value); -} - -// analysis_ftn_cb(hg_handle_t handle) -HG_TEST_RPC_CB(analysis_ftn, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - analysis_ftn_in_t in; - analysis_ftn_out_t out; - struct _pdc_region_analysis_ftn_info *thisFtn = NULL; - int nulliter_count = 0; - pdcid_t iterIn = -1, iterOut = -1; - pdcid_t registrationId = -1; - void * ftnHandle = NULL; - int (*ftnPtr)(pdcid_t, pdcid_t) = NULL; - int result; - - FUNC_ENTER(NULL); - - memset(&in, 0, sizeof(in)); - // Decode input - HG_Get_input(handle, &in); - - if (PDC_get_ftnPtr_(in.ftn_name, in.loadpath, &ftnHandle) < 0) - PGOTO_ERROR(FAIL, "PDC_get_ftnPtr_ returned an error"); - - if ((ftnPtr = ftnHandle) == NULL) - PGOTO_ERROR(FAIL, "Transforms function lookup failed"); - - if (ftnPtr != NULL) { - if ((iterIn = in.iter_in) == 0) { - nulliter_count = 1; - LOG_INFO("input is a NULL iterator\n"); - } - else if (execution_locus == SERVER_MEMORY) { - /* inputIter = &PDC_Block_iterator_cache[iterIn]; */ - } - if ((iterOut = in.iter_out) == 0) { - nulliter_count += 1; - LOG_INFO("output is a NULL iterator\n"); - } - - /* For the unusual case where both the input and output iterators - * are NULL, we can skip registering the function because - * we will only invoke the function ONCE (see below). - * Otherwise, go ahead and register... - */ - if (nulliter_count < 2) { - if ((thisFtn = (struct _pdc_region_analysis_ftn_info *)PDC_calloc( - sizeof(struct _pdc_region_analysis_ftn_info), 1)) != NULL) { - thisFtn->ftnPtr = (int (*)())ftnPtr; - thisFtn->n_args = 2; - thisFtn->object_id = (pdcid_t *)PDC_calloc(2, sizeof(pdcid_t)); - registrationId = PDC_add_analysis_ptr_to_registry_(thisFtn); - out.remote_ftn_id = registrationId; - } - else { - out.remote_ftn_id = registrationId; - PGOTO_ERROR(FAIL, "Unable to allocate storage for the analysis function"); - } - } - } - else { - out.remote_ftn_id = registrationId; - PGOTO_ERROR(FAIL, "Failed to resolve %s to a function pointer", in.ftn_name); - } - - HG_Respond(handle, NULL, NULL, &out); - - /* We NORMALLY don't call the actual function on a simple registration of - * the analysis. The only time that we do is when BOTH object data iterators - * are NULL. In this case, we will call the function EXACTLY this one time. - */ - if (ftnPtr && (nulliter_count == 2)) { - result = ftnPtr(iterIn, iterOut); - LOG_INFO("function call result was %d\n----------------\n", result); - - /* FIXME: - * We might consider adding the function result into - * thisFtn. Under that assumption, we might need to - * somehow add a way to notify the invoking client - * with an exception to handle computation errors... - */ - if (thisFtn) { - thisFtn->ftn_lastResult = result; - } - } - -done: - HG_Free_input(handle, &in); - HG_Destroy(handle); - - FUNC_LEAVE(ret_value); -} - -// obj_data_iterator_cb(hg_handle_t handle) -HG_TEST_RPC_CB(obj_data_iterator, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - obj_data_iterator_in_t in; - obj_data_iterator_out_t out; - - memset(&in, 0, sizeof(in)); - // Decode input - HG_Get_input(handle, &in); - // LOG_INFO("obj_data_iterator_cb entered\n"); - ret_value = PDC_Server_instantiate_data_iterator(&in, &out); - - HG_Respond(handle, NULL, NULL, &out); - - HG_Free_input(handle, &in); - HG_Destroy(handle); - - FUNC_LEAVE(ret_value); -} - -HG_TEST_THREAD_CB(obj_data_iterator) -HG_TEST_THREAD_CB(analysis_ftn) - -hg_id_t -PDC_analysis_ftn_register(hg_class_t *hg_class) -{ - FUNC_ENTER(NULL); - - hg_id_t ret_value; - ret_value = - MERCURY_REGISTER(hg_class, "analysis_ftn", analysis_ftn_in_t, analysis_ftn_out_t, analysis_ftn_cb); - - FUNC_LEAVE(ret_value); -} - -hg_id_t -PDC_obj_data_iterator_register(hg_class_t *hg_class) -{ - FUNC_ENTER(NULL); - - hg_id_t ret_value; - ret_value = MERCURY_REGISTER(hg_class, "obj_data_iterator", obj_data_iterator_in_t, - obj_data_iterator_out_t, obj_data_iterator_cb); - - FUNC_LEAVE(ret_value); -} - -void -PDC_free_analysis_registry() -{ - FUNC_ENTER(NULL); - - int index; - if (pdc_region_analysis_registry && (registered_analysis_ftn_count_g > 0)) { - for (index = 0; index < registered_analysis_ftn_count_g; index++) { - pdc_region_analysis_registry[index] = - (struct _pdc_region_analysis_ftn_info *)PDC_free(pdc_region_analysis_registry[index]); - } - pdc_region_analysis_registry = - (struct _pdc_region_analysis_ftn_info **)PDC_free(pdc_region_analysis_registry); - } - - FUNC_LEAVE_VOID(); -} - -void -PDC_free_transform_registry() -{ - FUNC_ENTER(NULL); - - int index; - if (pdc_region_transform_registry && (registered_transform_ftn_count_g > 0)) { - for (index = 0; index < registered_transform_ftn_count_g; index++) { - pdc_region_transform_registry[index] = - (struct _pdc_region_transform_ftn_info *)PDC_free(pdc_region_transform_registry[index]); - } - pdc_region_transform_registry = - (struct _pdc_region_transform_ftn_info **)PDC_free(pdc_region_transform_registry); - } - - FUNC_LEAVE_VOID(); -} - -void -PDC_free_iterator_cache() -{ - FUNC_ENTER(NULL); - - if (PDC_Block_iterator_cache != NULL) - PDC_Block_iterator_cache = (struct _pdc_iterator_info *)PDC_free(PDC_Block_iterator_cache); - - FUNC_LEAVE_VOID(); -} diff --git a/src/api/pdc_client_connect.c b/src/api/pdc_client_connect.c index e2058fa6f..46216dd76 100644 --- a/src/api/pdc_client_connect.c +++ b/src/api/pdc_client_connect.c @@ -39,18 +39,20 @@ #include "pdc_cont.h" #include "pdc_region.h" #include "pdc_interface.h" -#include "pdc_analysis_pkg.h" -#include "pdc_transforms_common.h" #include "pdc_client_connect.h" #include "pdc_logger.h" #include "pdc_timing.h" #include "pdc_malloc.h" +#include "pdc_mercury_auth.h" #include "mercury.h" #include "mercury_macros.h" +#include "mercury_proc_string.h" #include "string_utils.h" #include "dart_core.h" +#include "pdc_tf.h" +#include "pdc_tf_common.h" #include "timer_utils.h" #include "query_utils.h" @@ -114,6 +116,97 @@ static hg_class_t * send_class_g = NULL; static hg_context_t *send_context_g = NULL; int query_id_g = 0; +static pbool_t +PDC_is_cxi_mercury_transport(const char *hg_transport) +{ + return (hg_transport != NULL && + (strcmp(hg_transport, "ofi+cxi") == 0 || strcmp(hg_transport, "cxi") == 0)); +} + +static pbool_t +PDC_is_perlmutter_system(void) +{ + const char *nersc_host = getenv("NERSC_HOST"); + + return (nersc_host != NULL && strcmp(nersc_host, "perlmutter") == 0); +} + +static int +PDC_get_local_process_rank(int fallback_rank) +{ + const char *env_names[] = {"SLURM_LOCALID", "MPI_LOCALRANKID", "PMI_LOCAL_RANK", + "OMPI_COMM_WORLD_LOCAL_RANK"}; + size_t i; + + for (i = 0; i < sizeof(env_names) / sizeof(env_names[0]); i++) { + char *endptr = NULL; + char *env_value = getenv(env_names[i]); + long local_rank = 0; + + if (env_value == NULL || env_value[0] == '\0') + continue; + + local_rank = strtol(env_value, &endptr, 10); + if (endptr != env_value && *endptr == '\0' && local_rank >= 0 && local_rank <= INT_MAX) + return (int)local_rank; + } + + return fallback_rank; +} + +static void +PDC_get_default_cxi_endpoint(char *host_buf, size_t host_buf_len, int fallback_rank, int pid_base) +{ + const char *nic_name = getenv("HG_CXI_NIC"); + int local_rank = PDC_get_local_process_rank(fallback_rank); + int pid_offset = local_rank % 255; + int pid = pid_base + pid_offset; + + if (nic_name == NULL || nic_name[0] == '\0') + nic_name = "cxi0"; + + snprintf(host_buf, host_buf_len, "%s:%d", nic_name, pid); +} + +static const char * +PDC_get_default_mercury_transport(void) +{ + return (PDC_is_perlmutter_system() == TRUE) ? "ofi+cxi" : "ofi+tcp"; +} + +static perr_t +PDC_build_mercury_na_info_string(const char *hg_transport, int port, int cxi_pid_base, char *na_info_string, + size_t na_info_string_len, char *host_buf, size_t host_buf_len) +{ + perr_t ret_value = SUCCEED; + const char *hostname = getenv("HG_HOST"); + + if (hostname == NULL || hostname[0] == '\0') { + if (PDC_is_cxi_mercury_transport(hg_transport) == TRUE) { + PDC_get_default_cxi_endpoint(host_buf, host_buf_len, pdc_client_mpi_rank_g, cxi_pid_base); + hostname = host_buf; + if (pdc_client_mpi_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); + } + else { + memset(host_buf, 0, host_buf_len); + gethostname(host_buf, host_buf_len - 1); + hostname = host_buf; + if (pdc_client_mpi_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); + } + } + else + LOG_INFO("Environment variable HG_HOST was set\n"); + + if (PDC_is_cxi_mercury_transport(hg_transport) == TRUE) + snprintf(na_info_string, na_info_string_len, "%s://%s", hg_transport, hostname); + else + snprintf(na_info_string, na_info_string_len, "%s://%s:%d", hg_transport, hostname, port); + + return ret_value; +} + // flags for RPC request and Bulk transfer. // When a work is put in the queue, increase todo_g by 1 // When a work is done and popped from the queue, decrease (todo_g) by 1. @@ -148,9 +241,7 @@ static hg_id_t metadata_del_kvtag_register_id_g; static hg_id_t metadata_get_kvtag_register_id_g; static hg_id_t region_lock_register_id_g; static hg_id_t region_release_register_id_g; -static hg_id_t transform_region_release_register_id_g; -static hg_id_t region_transform_release_register_id_g; -static hg_id_t region_analysis_release_register_id_g; + static hg_id_t data_server_read_register_id_g; static hg_id_t data_server_read_check_register_id_g; static hg_id_t data_server_write_check_register_id_g; @@ -371,7 +462,7 @@ PDC_Client_read_server_addr_from_file() FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; - int max_tries = 9, sleeptime = 1; + int max_tries = 9, sleeptime = 5; int i = 0, is_server_ready = 0; char * p; FILE * na_config = NULL; @@ -381,7 +472,7 @@ PDC_Client_read_server_addr_from_file() if (pdc_client_mpi_rank_g == 0) { sprintf(config_fname, "%s/%s", pdc_client_tmp_dir_g, pdc_server_cfg_name_g); - for (i = 0; i < max_tries; i++) { + for (;; /*i = 0; i < max_tries; i++*/) { if (access(config_fname, F_OK) != -1) { is_server_ready = 1; break; @@ -390,7 +481,7 @@ PDC_Client_read_server_addr_from_file() "waiting %d seconds\n", config_fname, sleeptime); sleep(sleeptime); - sleeptime *= 2; + // sleeptime *= 2; } if (is_server_ready != 1) PGOTO_ERROR(FAIL, "Server is not ready"); @@ -1231,11 +1322,10 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po { FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - char na_info_string[NA_STRING_INFO_LEN]; - char * hostname; - pbool_t free_hostname = false; - int local_server_id; + perr_t ret_value = SUCCEED; + char na_info_string[NA_STRING_INFO_LEN]; + char host_buf[HOSTNAME_LEN]; + int local_server_id; /* Set the default mercury transport * but enable overriding that to any of: @@ -1243,9 +1333,17 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po * "ofi+tcp" * "cci+tcp" */ - struct hg_init_info init_info = {0}; - char * default_hg_transport = "ofi+tcp"; - char * hg_transport; + struct hg_init_info init_info = {0}; + const char * default_hg_transport = PDC_get_default_mercury_transport(); + char * hg_transport; + int cxi_pid_base = 256; + pdc_scoped_env_entry_t scoped_envs[PDC_PERLMUTTER_CXI_ENV_COUNT]; + int scoped_env_count = 0; + unsigned int scoped_svc_id = 0; + unsigned int scoped_vni = 0; + char scoped_device[32]; + char * hostname = NULL; + bool free_hostname = false; #ifdef PDC_HAS_CRAY_DRC uint32_t credential, cookie; drc_info_handle_t credential_info; @@ -1255,11 +1353,11 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po #endif if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { - hg_transport = default_hg_transport; + hg_transport = (char *)default_hg_transport; if (pdc_client_mpi_rank_g == 0) LOG_INFO("Environment variable HG_TRANSPORT was NOT set, default to %s\n", default_hg_transport); } - else + else if (pdc_client_mpi_rank_g == 0) LOG_INFO("Environment variable HG_TRANSPORT was set\n"); if ((hostname = getenv("HG_HOST")) == NULL) { hostname = PDC_malloc(HOSTNAME_LEN); @@ -1269,22 +1367,22 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po if (pdc_client_mpi_rank_g == 0) LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); } - else + else if (pdc_client_mpi_rank_g == 0) LOG_INFO("Environment variable HG_HOST was set\n"); - sprintf(na_info_string, "%s://%s:%d", hg_transport, hostname, port); + ret_value = PDC_build_mercury_na_info_string(hg_transport, port, cxi_pid_base, na_info_string, + sizeof(na_info_string), host_buf, sizeof(host_buf)); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Error building Mercury connection string"); if (pdc_client_mpi_rank_g == 0) LOG_INFO("Connection string: %s\n", na_info_string); - if (free_hostname) - hostname = (char *)PDC_free(hostname); // gni starts here #ifdef PDC_HAS_CRAY_DRC /* Acquire credential */ - if (pdc_client_mpi_rank_g == 0) { + if (pdc_client_mpi_rank_g == 0) credential = atoi(getenv("PDC_DRC_KEY")); - } MPI_Bcast(&credential, 1, MPI_UINT32_T, 0, PDC_CLIENT_COMM_WORLD_g); rc = drc_access(credential, 0, &credential_info); @@ -1319,7 +1417,15 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po #ifdef PDC_HAS_SHARED_SERVER init_info.auto_sm = HG_TRUE; #endif + if (PDC_scope_perlmutter_cxi_auth_env(hg_transport, scoped_envs, &scoped_env_count, &scoped_svc_id, + &scoped_vni, scoped_device, sizeof(scoped_device)) != SUCCEED && + pdc_client_mpi_rank_g == 0) + LOG_WARNING("Unable to scope Slingshot CXI auth for Mercury, falling back to provider defaults"); + else if (scoped_env_count > 0 && pdc_client_mpi_rank_g == 0) + LOG_INFO("Scoped Slingshot CXI auth for Mercury: svc=%u vni=%u device=%s\n", scoped_svc_id, + scoped_vni, scoped_device); *hg_class = HG_Init_opt(na_info_string, HG_TRUE, &init_info); + PDC_restore_scoped_env(scoped_envs, scoped_env_count); if (*hg_class == NULL) PGOTO_ERROR(FAIL, "Error with HG_Init()"); @@ -1334,29 +1440,25 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po flush_obj_register_id_g = PDC_flush_obj_register(*hg_class); flush_obj_all_register_id_g = PDC_flush_obj_all_register(*hg_class); obj_reset_dims_register_id_g = PDC_obj_reset_dims_register(*hg_class); - // HG_Registered_disable_response(*hg_class, close_server_register_id_g, HG_TRUE); - - metadata_query_register_id_g = PDC_metadata_query_register(*hg_class); - container_query_register_id_g = PDC_container_query_register(*hg_class); - metadata_delete_register_id_g = PDC_metadata_delete_register(*hg_class); - metadata_delete_by_id_register_id_g = PDC_metadata_delete_by_id_register(*hg_class); - metadata_update_register_id_g = PDC_metadata_update_register(*hg_class); - metadata_add_tag_register_id_g = PDC_metadata_add_tag_register(*hg_class); - metadata_add_kvtag_register_id_g = PDC_metadata_add_kvtag_register(*hg_class); - metadata_del_kvtag_register_id_g = PDC_metadata_del_kvtag_register(*hg_class); - metadata_get_kvtag_register_id_g = PDC_metadata_get_kvtag_register(*hg_class); - region_lock_register_id_g = PDC_region_lock_register(*hg_class); - region_release_register_id_g = PDC_region_release_register(*hg_class); - transform_region_release_register_id_g = PDC_transform_region_release_register(*hg_class); - region_transform_release_register_id_g = PDC_region_transform_release_register(*hg_class); - region_analysis_release_register_id_g = PDC_region_analysis_release_register(*hg_class); - data_server_read_register_id_g = PDC_data_server_read_register(*hg_class); - data_server_read_check_register_id_g = PDC_data_server_read_check_register(*hg_class); - data_server_write_check_register_id_g = PDC_data_server_write_check_register(*hg_class); - data_server_write_register_id_g = PDC_data_server_write_register(*hg_class); - server_checkpoint_rpc_register_id_g = PDC_server_checkpoint_rpc_register(*hg_class); - send_shm_register_id_g = PDC_send_shm_register(*hg_class); - send_rpc_register_id_g = PDC_send_rpc_register(*hg_class); + + metadata_query_register_id_g = PDC_metadata_query_register(*hg_class); + container_query_register_id_g = PDC_container_query_register(*hg_class); + metadata_delete_register_id_g = PDC_metadata_delete_register(*hg_class); + metadata_delete_by_id_register_id_g = PDC_metadata_delete_by_id_register(*hg_class); + metadata_update_register_id_g = PDC_metadata_update_register(*hg_class); + metadata_add_tag_register_id_g = PDC_metadata_add_tag_register(*hg_class); + metadata_add_kvtag_register_id_g = PDC_metadata_add_kvtag_register(*hg_class); + metadata_del_kvtag_register_id_g = PDC_metadata_del_kvtag_register(*hg_class); + metadata_get_kvtag_register_id_g = PDC_metadata_get_kvtag_register(*hg_class); + region_lock_register_id_g = PDC_region_lock_register(*hg_class); + region_release_register_id_g = PDC_region_release_register(*hg_class); + data_server_read_register_id_g = PDC_data_server_read_register(*hg_class); + data_server_read_check_register_id_g = PDC_data_server_read_check_register(*hg_class); + data_server_write_check_register_id_g = PDC_data_server_write_check_register(*hg_class); + data_server_write_register_id_g = PDC_data_server_write_register(*hg_class); + server_checkpoint_rpc_register_id_g = PDC_server_checkpoint_rpc_register(*hg_class); + send_shm_register_id_g = PDC_send_shm_register(*hg_class); + send_rpc_register_id_g = PDC_send_rpc_register(*hg_class); // bulk query_partial_register_id_g = PDC_query_partial_register(*hg_class); @@ -1379,11 +1481,6 @@ PDC_Client_mercury_init(hg_class_t **hg_class, hg_context_t **hg_context, int po buf_map_register_id_g = PDC_buf_map_register(*hg_class); buf_unmap_register_id_g = PDC_buf_unmap_register(*hg_class); - // Analysis and Transforms - analysis_ftn_register_id_g = PDC_analysis_ftn_register(*hg_class); - transform_ftn_register_id_g = PDC_transform_ftn_register(*hg_class); - object_data_iterator_register_id_g = PDC_obj_data_iterator_register(*hg_class); - PDC_server_lookup_client_register(*hg_class); PDC_notify_io_complete_register(*hg_class); PDC_notify_region_update_register(*hg_class); @@ -1514,8 +1611,6 @@ PDC_Client_init() if (pdc_nclient_per_server_g <= 0) pdc_nclient_per_server_g = 1; - PDC_set_execution_locus(CLIENT_MEMORY); - if (pdc_client_mpi_rank_g == 0) { LOG_INFO("Found %d PDC Metadata servers, running with %d PDC clients\n", pdc_server_num_g, pdc_client_mpi_size_g); @@ -2185,18 +2280,6 @@ PDC_Client_update_metadata(pdc_metadata_t *old, pdc_metadata_t *new) in.new_metadata.dims2 = new->dims[2]; in.new_metadata.dims3 = new->dims[3]; - // New fields to support transform state changes - // and possibly provenance info. - in.new_metadata.current_state = new->transform_state; - in.new_metadata.t_storage_order = new->current_state.storage_order; - in.new_metadata.t_dtype = new->current_state.dtype; - in.new_metadata.t_ndim = new->current_state.ndim; - in.new_metadata.t_dims0 = new->current_state.dims[0]; - in.new_metadata.t_dims1 = new->current_state.dims[1]; - in.new_metadata.t_dims2 = new->current_state.dims[2]; - in.new_metadata.t_dims3 = new->current_state.dims[3]; - in.new_metadata.t_meta_index = new->current_state.meta_index; - hg_ret = HG_Forward(metadata_update_handle, metadata_update_rpc_cb, &lookup_args, &in); if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not start HG_Forward"); @@ -3237,7 +3320,6 @@ PDC_Client_transfer_request_wait_all(hg_bulk_t *bulk_handle, int n_objs, pdcid_t in.total_buf_size = sizeof(pdcid_t) * n_objs; // Compute metadata server id - debug_server_id_count[data_server_id]++; hg_class = HG_Context_get_class(send_context_g); @@ -3255,7 +3337,6 @@ PDC_Client_transfer_request_wait_all(hg_bulk_t *bulk_handle, int n_objs, pdcid_t *bulk_handle = in.local_bulk_handle; if (hg_ret != HG_SUCCESS) PGOTO_ERROR(FAIL, "Could not create local bulk data handle"); - hg_ret = HG_Forward(client_send_transfer_request_wait_all_handle, client_send_transfer_request_wait_all_rpc_cb, &transfer_args, &in); @@ -3267,7 +3348,6 @@ PDC_Client_transfer_request_wait_all(hg_bulk_t *bulk_handle, int n_objs, pdcid_t PGOTO_ERROR(FAIL, "Could not start HG_Forward"); hg_atomic_set32(&atomic_work_todo_g, 1); PDC_Client_check_response(&send_context_g); - if (transfer_args.ret != 1) PGOTO_ERROR(FAIL, "Transfer request wait all failed"); @@ -3287,7 +3367,7 @@ perr_t PDC_Client_transfer_request(hg_bulk_t *bulk_handle, void *buf, pdcid_t obj_id, uint32_t data_server_id, int obj_ndim, uint64_t *obj_dims, int remote_ndim, uint64_t *remote_offset, uint64_t *remote_size, size_t unit, pdc_access_t access_type, - pdcid_t *metadata_id) + pdcid_t *metadata_id, struct _pdc_obj_info *obj_pointer) { FUNC_ENTER(NULL); @@ -3301,6 +3381,7 @@ PDC_Client_transfer_request(hg_bulk_t *bulk_handle, void *buf, pdcid_t obj_id, u hg_handle_t client_send_transfer_request_handle; struct _pdc_transfer_request_args transfer_args; char cur_time[64]; + pdc_tf_region_mapping_t * region_mapping = NULL; FUNC_ENTER(NULL); #ifdef PDC_TIMING @@ -3311,13 +3392,13 @@ PDC_Client_transfer_request(hg_bulk_t *bulk_handle, void *buf, pdcid_t obj_id, u PGOTO_ERROR(FAIL, "Invalid PDC type"); LOG_DEBUG("rank = %d, data_server_id = %u\n", pdc_client_mpi_rank_g, data_server_id); - in.access_type = access_type; - in.remote_unit = unit; - in.obj_id = obj_id; + in.access_type = access_type; + in.remote_unit = unit; + in.obj_id = obj_id; + in.pdc_tf_pkg.pdc_var_type = PDC_UNKNOWN; in.obj_ndim = obj_ndim; PDC_copy_region_desc(obj_dims, in.obj_dims, in.obj_ndim, in.obj_ndim); - // Compute metadata server id meta_server_id = PDC_get_server_by_obj_id(obj_id, pdc_server_num_g); @@ -3337,6 +3418,46 @@ PDC_Client_transfer_request(hg_bulk_t *bulk_handle, void *buf, pdcid_t obj_id, u if (PDC_Client_try_lookup_server(data_server_id, 0) != SUCCEED) PGOTO_ERROR(FAIL, "Error with PDC_Client_try_lookup_server"); + // Checked for transformations associated with region + if (access_type == PDC_WRITE && obj_pointer != NULL && obj_pointer->pdc_tf_obj != NULL && + PDCtf_region_has_attached_graph(obj_pointer->pdc_tf_obj, remote_ndim, unit, remote_offset, + remote_size, ®ion_mapping)) { + + LOG_DEBUG("Region transfer as an attached graph\n"); + pdcid_t dg_id = region_mapping->region_state.dg_id; + pdc_dg_t *dg = PDCtf_get_dg(dg_id); + + if (dg == NULL) + PGOTO_ERROR(FAIL, "dg was NULL"); + + char *json_filepath = (char *)dg->data; + + LOG_DEBUG("Region transfer json filepath: %s\n", json_filepath); + LOG_DEBUG("Region transfer current state: %s\n", region_mapping->region_state.cur_state); + LOG_DEBUG("Region transfer store state: %s\n", region_mapping->region_state.store_state); + LOG_DEBUG("Region transfer client state: %s\n", region_mapping->region_state.client_state); + + in.pdc_tf_pkg.pdc_var_type = obj_pointer->obj_pt->obj_prop_pub->type; + in.pdc_tf_pkg.json_filepath = json_filepath; + in.pdc_tf_pkg.store_state = (hg_string_t)region_mapping->region_state.store_state; + in.pdc_tf_pkg.client_state = (hg_string_t)region_mapping->region_state.client_state; + + if (access_type == PDC_WRITE) + in.pdc_tf_pkg.cur_state = (hg_string_t)region_mapping->region_state.client_state; + else if (access_type == PDC_READ) + in.pdc_tf_pkg.cur_state = (hg_string_t)region_mapping->region_state.store_state; + else + PGOTO_ERROR(FAIL, "Invalid access type for transformation"); + } + else { + LOG_DEBUG("Region transfer does NOT have an attached graph\n"); + + in.pdc_tf_pkg.json_filepath = (hg_string_t)NULL; + in.pdc_tf_pkg.store_state = (hg_string_t)NULL; + in.pdc_tf_pkg.cur_state = (hg_string_t)NULL; + in.pdc_tf_pkg.client_state = (hg_string_t)NULL; + } + hg_ret = HG_Create(send_context_g, pdc_server_info_g[data_server_id].addr, transfer_request_register_id_g, &client_send_transfer_request_handle); @@ -3444,13 +3565,6 @@ PDC_Client_transfer_request_wait(pdcid_t transfer_request_id, uint32_t data_serv struct _pdc_transfer_request_wait_args transfer_args; char cur_time[64]; - // Join the thread of trasfer start - /* if (hg_progress_flag_g == 0) { */ - /* hg_progress_flag_g = 1; */ - /* pthread_join(hg_progress_tid_g, NULL); */ - /* hg_progress_flag_g = -1; */ - /* } */ - #ifdef PDC_TIMING double start = MPI_Wtime(), end; double function_start = start; @@ -4140,7 +4254,6 @@ PDC_Client_data_server_read(struct pdc_request *request) if (meta == NULL || region == NULL) PGOTO_ERROR(FAIL, "Invalid metadata or region"); - // TODO TEMPWORK char *tmp_env = getenv("PDC_CACHE_PERCENTAGE"); int cache_percentage; if (tmp_env != NULL) diff --git a/src/api/pdc_analysis/include/pdc_hist_pkg.h b/src/api/pdc_hist/include/pdc_hist_pkg.h similarity index 100% rename from src/api/pdc_analysis/include/pdc_hist_pkg.h rename to src/api/pdc_hist/include/pdc_hist_pkg.h diff --git a/src/api/pdc_analysis/pdc_hist_pkg.c b/src/api/pdc_hist/pdc_hist_pkg.c similarity index 100% rename from src/api/pdc_analysis/pdc_hist_pkg.c rename to src/api/pdc_hist/pdc_hist_pkg.c diff --git a/src/api/pdc_obj/include/pdc_obj_pkg.h b/src/api/pdc_obj/include/pdc_obj_pkg.h index 11effda8e..13093c61e 100644 --- a/src/api/pdc_obj/include/pdc_obj_pkg.h +++ b/src/api/pdc_obj/include/pdc_obj_pkg.h @@ -51,6 +51,7 @@ struct _pdc_obj_info { pdc_local_transfer_request *local_transfer_request_head; pdc_local_transfer_request *local_transfer_request_end; int local_transfer_request_size; + struct pdc_tf_obj_t * pdc_tf_obj; }; /***************************************/ diff --git a/src/api/pdc_obj/include/pdc_prop_pkg.h b/src/api/pdc_obj/include/pdc_prop_pkg.h index 16524314b..c88b5a756 100644 --- a/src/api/pdc_obj/include/pdc_prop_pkg.h +++ b/src/api/pdc_obj/include/pdc_prop_pkg.h @@ -35,16 +35,6 @@ struct _pdc_cont_prop { pdc_lifetime_t cont_life; }; -typedef enum { ROW_major, COL_major } _pdc_major_type_t; - -struct _pdc_transform_state { - _pdc_major_type_t storage_order; - pdc_var_type_t dtype; - size_t ndim; - uint64_t dims[4]; - int meta_index; /* transform to this state */ -}; - struct _pdc_obj_prop { struct pdc_obj_prop *obj_prop_pub; struct _pdc_class * pdc; @@ -55,12 +45,6 @@ struct _pdc_obj_prop { char * tags; void * buf; pdc_kvtag_t * kvtag; - - /* The following have been added to support of PDC analysis and transforms */ - size_t type_extent; - uint64_t locus; - uint32_t data_state; - struct _pdc_transform_state transform_prop; }; /***************************************/ diff --git a/src/api/pdc_obj/pdc_obj.c b/src/api/pdc_obj/pdc_obj.c index c32e9272a..41cb4357b 100644 --- a/src/api/pdc_obj/pdc_obj.c +++ b/src/api/pdc_obj/pdc_obj.c @@ -32,10 +32,10 @@ #include "pdc_obj_pkg.h" #include "pdc_obj.h" #include "pdc_interface.h" -#include "pdc_transforms_pkg.h" -#include "pdc_analysis_pkg.h" #include "pdc_client_connect.h" #include "pdc_logger.h" +#include "pdc_tf.h" + #include #include #include @@ -61,7 +61,20 @@ pdcid_t PDCobj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id) { FUNC_ENTER(NULL); - FUNC_LEAVE(PDC_obj_create(cont_id, obj_name, obj_prop_id, PDC_OBJ_GLOBAL)); + + pdcid_t ret_value = 0; + + ret_value = PDC_obj_create(cont_id, obj_name, obj_prop_id, PDC_OBJ_GLOBAL); + if (ret_value == 0) + PGOTO_ERROR(0, "Failed to obj_create\n"); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + if (dg_id == 0) + PGOTO_ERROR(0, "Failed to dg json create\n"); + if (PDCtf_attach_to_obj(dg_id, ret_value, "decompressed", "compressed") != SUCCEED) + PGOTO_ERROR(0, "Failed to attach_to_obj\n"); + +done: + FUNC_LEAVE(ret_value); } /** @@ -125,12 +138,13 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ size_t i; perr_t ret = SUCCEED; - p = (struct _pdc_obj_info *)PDC_malloc(sizeof(struct _pdc_obj_info)); + p = (struct _pdc_obj_info *)PDC_calloc(1, sizeof(struct _pdc_obj_info)); if (!p) PGOTO_ERROR(0, "PDC object memory allocation failed"); p->metadata = NULL; p->location = location; p->region_list_head = NULL; + p->pdc_tf_obj = NULL; if (cont_id == 0) { meta_id = 0; @@ -185,7 +199,6 @@ PDC_obj_create(pdcid_t cont_id, const char *obj_name, pdcid_t obj_prop_id, _pdc_ p->obj_pt->data_loc = strdup(obj_prop->data_loc); if (obj_prop->tags) p->obj_pt->tags = strdup(obj_prop->tags); - p->obj_pt->locus = PDC_get_execution_locus(); p->obj_pt->pdc = (struct _pdc_class *)PDC_calloc(1, sizeof(struct _pdc_class)); if (!p->obj_pt->pdc) @@ -423,7 +436,7 @@ PDCobj_open_common(const char *obj_name, pdcid_t pdc, int is_col) PGOTO_DONE(info->local_id); } - p = (struct _pdc_obj_info *)PDC_malloc(sizeof(struct _pdc_obj_info)); + p = (struct _pdc_obj_info *)PDC_calloc(1, sizeof(struct _pdc_obj_info)); if (!p) PGOTO_ERROR(0, "PDC object memory allocation failed"); p->cont = (struct _pdc_cont_info *)PDC_calloc(1, sizeof(struct _pdc_cont_info)); @@ -484,15 +497,6 @@ PDCobj_open_common(const char *obj_name, pdcid_t pdc, int is_col) p->obj_pt->time_step = out->time_step; p->obj_pt->user_id = out->user_id; - if (out->transform_state > 0) { - p->obj_pt->locus = SERVER_MEMORY; - p->obj_pt->data_state = out->transform_state; - p->obj_pt->transform_prop.storage_order = out->current_state.storage_order; - p->obj_pt->transform_prop.dtype = out->current_state.dtype; - p->obj_pt->transform_prop.ndim = out->current_state.ndim; - for (i = 0; i < out->current_state.ndim; i++) - p->obj_pt->transform_prop.dims[i] = out->current_state.dims[i]; - } p->metadata = out; p->local_transfer_request_head = NULL; @@ -1002,7 +1006,7 @@ PDC_obj_get_info(pdcid_t obj_id) PGOTO_ERROR(NULL, "cannot allocate ret_value->obj_pt"); ret_value->obj_pt->obj_prop_pub->dims = PDC_malloc(ret_value->obj_pt->obj_prop_pub->ndim * sizeof(uint64_t)); - LOG_INFO("ndim is %d\n", ret_value->obj_pt->obj_prop_pub->ndim); + LOG_DEBUG("ndim is %d\n", ret_value->obj_pt->obj_prop_pub->ndim); if (ret_value->obj_pt->obj_prop_pub->ndim == 0 || ret_value->obj_pt->obj_prop_pub->dims) { for (i = 0; i < ret_value->obj_pt->obj_prop_pub->ndim; i++) { ret_value->obj_pt->obj_prop_pub->dims[i] = info->obj_pt->obj_prop_pub->dims[i]; @@ -1089,9 +1093,6 @@ PDCobj_get_info(pdcid_t obj_id) struct pdc_obj_info * ret_value = NULL; struct _pdc_obj_info *tmp = NULL; - /* pdcid_t obj_id; */ - - /* obj_id = PDC_find_byname(PDC_OBJ, obj_name); */ tmp = PDC_obj_get_info(obj_id); if (NULL == tmp) diff --git a/src/api/pdc_obj/pdc_prop.c b/src/api/pdc_obj/pdc_prop.c index eb6ad9ab4..98946d8e4 100644 --- a/src/api/pdc_obj/pdc_prop.c +++ b/src/api/pdc_obj/pdc_prop.c @@ -96,7 +96,7 @@ PDCprop_create(pdc_prop_type_t type, pdcid_t pdcid) q->obj_prop_pub->ndim = 1; q->obj_prop_pub->dims = (uint64_t *)PDC_calloc(1, q->obj_prop_pub->ndim * sizeof(uint64_t)); q->obj_prop_pub->type = PDC_UNKNOWN; - q->obj_prop_pub->region_partition = PDC_REGION_STATIC; + q->obj_prop_pub->region_partition = PDC_OBJ_STATIC; q->obj_prop_pub->consistency = PDC_CONSISTENCY_EVENTUAL; q->data_loc = NULL; q->app_name = NULL; @@ -114,10 +114,6 @@ PDCprop_create(pdc_prop_type_t type, pdcid_t pdcid) if (pdc_class->name) q->pdc->name = strdup(pdc_class->name); q->pdc->local_id = pdc_class->local_id; - q->type_extent = 0; - q->data_state = 0; - q->locus = CLIENT_MEMORY; - memset(&q->transform_prop, 0, sizeof(struct _pdc_transform_state)); ret_value = new_id_o; } diff --git a/src/api/pdc_region/pdc_region.c b/src/api/pdc_region/pdc_region.c index e6fba8721..8efcc1ef3 100644 --- a/src/api/pdc_region/pdc_region.c +++ b/src/api/pdc_region/pdc_region.c @@ -36,9 +36,7 @@ #include "pdc_region_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_interface.h" -#include "pdc_transforms_pkg.h" #include "pdc_client_connect.h" -#include "pdc_analysis_pkg.h" #include static perr_t pdc_region_close(struct pdc_region_info *op); @@ -216,12 +214,6 @@ PDCbuf_obj_map(void *buf, pdc_var_type_t local_type, pdcid_t local_reg, pdcid_t local_type, buf, remote_type, reg1, reg2, obj2); if (ret_value == SUCCEED) { - /* - * For analysis and/or transforms, we only identify the target region as being mapped. - * The lock/unlock protocol for writing will protect the target from being written by - * more than one source. - */ - PDC_check_transform(PDC_DATA_MAP, reg2); PDC_inc_ref(remote_obj); PDC_inc_ref(remote_reg); } diff --git a/src/api/pdc_region/pdc_region_transfer.c b/src/api/pdc_region/pdc_region_transfer.c index dda936f02..167767471 100644 --- a/src/api/pdc_region/pdc_region_transfer.c +++ b/src/api/pdc_region/pdc_region_transfer.c @@ -39,10 +39,10 @@ #include "pdc_region_pkg.h" #include "pdc_obj_pkg.h" #include "pdc_interface.h" -#include "pdc_transforms_pkg.h" #include "pdc_client_connect.h" -#include "pdc_analysis_pkg.h" #include "pdc_logger.h" +#include "pdc_tf.h" +#include "pdc_tf_common.h" #include #define PDC_MERGE_TRANSFER_MIN_COUNT 50 @@ -134,8 +134,7 @@ typedef struct pdc_transfer_request_start_all_pkg { // Every time a start_all pkg is created, this index is set to be the index such that the data_server_id // appears in the transfer_request->obj_servers. int index; - // Data buffer. This data buffer is contiguous according to the remote region. We assume this is after - // transformation of local regions + // Data buffer. This data buffer is contiguous according to the remote region. char * buf; struct pdc_transfer_request_start_all_pkg *next; } pdc_transfer_request_start_all_pkg; @@ -286,8 +285,8 @@ PDCregion_transfer_create(void *buf, pdc_access_t access_type, pdcid_t obj_id, p FUNC_ENTER(NULL); pdcid_t ret_value = SUCCEED; - struct _pdc_id_info * objinfo2; - struct _pdc_obj_info * obj2; + struct _pdc_id_info * obj_info; + struct _pdc_obj_info * obj; pdc_transfer_request * p; struct _pdc_id_info * reginfo1, *reginfo2; struct pdc_region_info *reg1, *reg2; @@ -303,38 +302,52 @@ PDCregion_transfer_create(void *buf, pdc_access_t access_type, pdcid_t obj_id, p if ((reginfo2 = PDC_find_id(remote_reg)) == NULL) PGOTO_ERROR(FAIL, "Failed to find PDC ID: %d", remote_reg); reg2 = (struct pdc_region_info *)(reginfo2->obj_ptr); - if ((objinfo2 = PDC_find_id(obj_id)) == NULL) + if ((obj_info = PDC_find_id(obj_id)) == NULL) PGOTO_ERROR(FAIL, "Failed to find PDC ID: %d", obj_id); - - obj2 = (struct _pdc_obj_info *)(objinfo2->obj_ptr); - - p = (pdc_transfer_request *)PDC_malloc(sizeof(pdc_transfer_request)); - p->obj_pointer = obj2; - p->mem_type = obj2->obj_pt->obj_prop_pub->type; - p->local_obj_id = obj_id; - p->obj_id = obj2->obj_info_pub->meta_id; - p->access_type = access_type; - p->buf = buf; - p->metadata_id = NULL; - p->read_bulk_buf = NULL; - p->new_buf = NULL; - p->bulk_buf = NULL; - p->bulk_buf_ref = NULL; - p->output_buf = NULL; - p->region_partition = ((pdc_metadata_t *)obj2->metadata)->region_partition; - // p->region_partition = PDC_REGION_LOCAL; - p->data_server_id = ((pdc_metadata_t *)obj2->metadata)->data_server_id; - p->metadata_server_id = obj2->obj_info_pub->metadata_server_id; + obj = (struct _pdc_obj_info *)(obj_info->obj_ptr); + + p = (pdc_transfer_request *)PDC_malloc(sizeof(pdc_transfer_request)); + p->obj_pointer = obj; + p->mem_type = obj->obj_pt->obj_prop_pub->type; + p->local_obj_id = obj_id; + p->obj_id = obj->obj_info_pub->meta_id; + p->access_type = access_type; + p->buf = buf; + p->metadata_id = NULL; + p->read_bulk_buf = NULL; + p->new_buf = NULL; + p->bulk_buf = NULL; + p->bulk_buf_ref = NULL; + p->output_buf = NULL; + p->region_partition = ((pdc_metadata_t *)obj->metadata)->region_partition; + p->data_server_id = ((pdc_metadata_t *)obj->metadata)->data_server_id; + p->metadata_server_id = obj->obj_info_pub->metadata_server_id; p->unit = PDC_get_var_type_size(p->mem_type); - p->consistency = obj2->obj_pt->obj_prop_pub->consistency; + p->consistency = obj->obj_pt->obj_prop_pub->consistency; p->merged_request_id = 0; p->is_done = 0; - PDCregion_transfer_init_bulk_handles(p); + + // Check if graph is attached to object + if (access_type == PDC_WRITE && obj->pdc_tf_obj != NULL && obj->pdc_tf_obj->attach_to_all_regions) { + pdcid_t dg_id = obj->pdc_tf_obj->all_regions_state.dg_id; + char * client_state = obj->pdc_tf_obj->all_regions_state.client_state; + char * store_state = obj->pdc_tf_obj->all_regions_state.store_state; + + assert(client_state != NULL); + assert(store_state != NULL); + + if (PDCtf_attach_to_region(dg_id, obj_id, remote_reg, client_state, store_state) != SUCCEED) + PGOTO_ERROR(0, "Failed to attach graph to region"); + } + + if (PDCregion_transfer_init_bulk_handles(p) != SUCCEED) + PGOTO_ERROR(0, "Error with PDCregion_transfer_init_bulk_handles"); + unit = p->unit; p->local_region_ndim = reg1->ndim; p->local_region_offset = (uint64_t *)PDC_malloc( - sizeof(uint64_t) * (reg1->ndim * 2 + reg2->ndim * 2 + obj2->obj_pt->obj_prop_pub->ndim)); + sizeof(uint64_t) * (reg1->ndim * 2 + reg2->ndim * 2 + obj->obj_pt->obj_prop_pub->ndim)); ptr = p->local_region_offset; memcpy(p->local_region_offset, reg1->offset, sizeof(uint64_t) * reg1->ndim); ptr += reg1->ndim; @@ -351,9 +364,9 @@ PDCregion_transfer_create(void *buf, pdc_access_t access_type, pdcid_t obj_id, p memcpy(p->remote_region_size, reg2->size, sizeof(uint64_t) * reg2->ndim); ptr += reg2->ndim; - p->obj_ndim = obj2->obj_pt->obj_prop_pub->ndim; + p->obj_ndim = obj->obj_pt->obj_prop_pub->ndim; p->obj_dims = ptr; - memcpy(p->obj_dims, obj2->obj_pt->obj_prop_pub->dims, sizeof(uint64_t) * p->obj_ndim); + memcpy(p->obj_dims, obj->obj_pt->obj_prop_pub->dims, sizeof(uint64_t) * p->obj_ndim); p->total_data_size = unit; for (j = 0; j < (int)reg2->ndim; ++j) { @@ -477,7 +490,7 @@ remove_local_transfer_request(struct _pdc_obj_info *p, pdcid_t transfer_request_ } /* - * Input: Ojbect dimensions + a region + * Input: Object dimensions + a region * Output: Data servers that the region will access with a static region partition. As well as overlapping * regions. */ @@ -684,6 +697,8 @@ pack_region_metadata_query(pdc_transfer_request_start_all_pkg **transfer_request { FUNC_ENTER(NULL); + LOG_DEBUG("pack_region_metadata_query was called!\n"); + perr_t ret_value = SUCCEED; int i; char * ptr; @@ -732,6 +747,8 @@ unpack_region_metadata_query(char *buf, pdc_transfer_request_start_all_pkg **tra { FUNC_ENTER(NULL); + LOG_DEBUG("unpack_region_metadata_query was called!\n"); + perr_t ret_value = SUCCEED; pdc_transfer_request_start_all_pkg *transfer_request_head, *transfer_request_end; pdc_transfer_request * local_request; @@ -906,6 +923,7 @@ register_metadata(pdc_transfer_request_start_all_pkg **transfer_request_input, i PDCregion_transfer_add_bulk_handle(transfer_requests[index]->transfer_request, bulk_handle); unpack_region_metadata_query(output_buf, transfer_requests + index, &transfer_request_head, &transfer_request_end, &output_size); + LOG_INFO("AFTER UNPACK %d\n", transfer_requests[index]->remote_offset[0]); output_buf = (char *)PDC_free(output_buf); if (transfer_request_front_head) @@ -1175,6 +1193,38 @@ finish_start_all_requests(pdc_transfer_request_start_all_pkg **write_transfer_re FUNC_LEAVE(0); } +// Helper function to compute the extra size needed for attached graph strings +static size_t +PDCcompute_region_transformation_metadata_size(struct _pdc_obj_info *obj_pointer, int remote_ndim, + size_t unit, uint64_t *remote_offset, uint64_t *remote_size, + pdc_access_t access_type) +{ + FUNC_ENTER(NULL); + + // at least terminators for cur_state, client_state, store_state, json filepath + size_t ret_value = 4; + pdc_tf_region_mapping_t *region_mapping = NULL; + + if (access_type == PDC_READ || !obj_pointer || !obj_pointer->pdc_tf_obj || + !PDCtf_region_has_attached_graph(obj_pointer->pdc_tf_obj, remote_ndim, unit, remote_offset, + remote_size, ®ion_mapping)) { + PGOTO_DONE(ret_value); + } + + pdc_dg_t *dg = PDCtf_get_dg(region_mapping->region_state.dg_id); + if (dg == NULL) + PGOTO_ERROR(ret_value, "Failed to PDCtf_get_dg in PDCcompute_region_transformation_metadata_size\n"); + // json filepath + ret_value += strlen((char *)dg->data); // add the length (we already counted +1 above) + // current, client, store state + ret_value += strlen(region_mapping->region_state.cur_state); + ret_value += strlen(region_mapping->region_state.client_state); + ret_value += strlen(region_mapping->region_state.store_state); + +done: + FUNC_LEAVE(ret_value); +} + static perr_t PDC_Client_pack_all_requests(int n_objs, pdc_transfer_request_start_all_pkg **transfer_requests, pdc_access_t access_type, char **bulk_buf_ptr, size_t *total_buf_size_ptr, @@ -1197,8 +1247,23 @@ PDC_Client_pack_all_requests(int n_objs, pdc_transfer_request_start_all_pkg **tr * obj_ndim: sizeof(int) * remote remote_ndim: sizeof(int) * unit: sizeof(size_t) + * var_type: sizeof(pdc_var_type_t) + * transformation strings lengths */ - metadata_size = n_objs * (sizeof(pdcid_t) + sizeof(int) * 2 + sizeof(size_t)); + metadata_size = n_objs * (sizeof(pdcid_t) + sizeof(int) * 2 + sizeof(size_t) + sizeof(pdc_var_type_t)); + for (i = 0; i < n_objs; ++i) { + struct _pdc_obj_info *obj_pointer = transfer_requests[i]->transfer_request->obj_pointer; + int remote_ndim = transfer_requests[i]->transfer_request->remote_region_ndim; + unit = transfer_requests[i]->transfer_request->unit; + uint64_t *offset = transfer_requests[i]->transfer_request->remote_region_offset; + uint64_t *size = transfer_requests[i]->transfer_request->remote_region_size; + + assert(obj_pointer != NULL); + + metadata_size += PDCcompute_region_transformation_metadata_size(obj_pointer, remote_ndim, unit, + offset, size, access_type); + } + // Data size, including region offsets/length pairs and actual data for I/O. /* * For each of objects @@ -1238,18 +1303,72 @@ PDC_Client_pack_all_requests(int n_objs, pdc_transfer_request_start_all_pkg **tr *bulk_buf_ptr = bulk_buf; ptr = bulk_buf; ptr2 = bulk_buf; + // Pack metadata #define MEMCPY_INC(a, b) \ { \ memcpy(ptr, a, b); \ ptr += b; \ } + for (i = 0; i < n_objs; ++i) { unit = transfer_requests[i]->transfer_request->unit; MEMCPY_INC(&(transfer_requests[i]->transfer_request->obj_id), sizeof(pdcid_t)); MEMCPY_INC(&(transfer_requests[i]->transfer_request->obj_ndim), sizeof(int)); MEMCPY_INC(&(transfer_requests[i]->transfer_request->remote_region_ndim), sizeof(int)); MEMCPY_INC(&unit, sizeof(size_t)); + pdc_var_type_t var_type = + transfer_requests[i]->transfer_request->obj_pointer->obj_pt->obj_prop_pub->type; + MEMCPY_INC(&var_type, sizeof(pdc_var_type_t)); + + LOG_DEBUG("obj_id: %d, ndim: %d, remote_region_ndim: %d, unit: %zu, var_type: %d\n", + transfer_requests[i]->transfer_request->obj_id, + transfer_requests[i]->transfer_request->obj_ndim, + transfer_requests[i]->transfer_request->remote_region_ndim, unit, + transfer_requests[i]->transfer_request->obj_pointer->obj_pt->obj_prop_pub->type); + + // Checked for transformations associated with region + pdc_tf_region_mapping_t *region_mapping = NULL; + struct _pdc_obj_info * obj_pointer = transfer_requests[i]->transfer_request->obj_pointer; + int remote_ndim = transfer_requests[i]->transfer_request->remote_region_ndim; + uint64_t * remote_offset = transfer_requests[i]->transfer_request->remote_region_offset; + uint64_t * remote_size = transfer_requests[i]->transfer_request->remote_region_size; + + const char *json_filepath = ""; + const char *cur_state_str = ""; + const char *client_state_str = ""; + const char *store_state_str = ""; + + if (access_type == PDC_WRITE && obj_pointer != NULL && obj_pointer->pdc_tf_obj != NULL && + PDCtf_region_has_attached_graph(obj_pointer->pdc_tf_obj, remote_ndim, unit, remote_offset, + remote_size, ®ion_mapping)) { + pdcid_t dg_id = region_mapping->region_state.dg_id; + pdc_dg_t *dg = PDCtf_get_dg(dg_id); + + assert(dg != NULL); + + json_filepath = (char *)dg->data; + + LOG_DEBUG("Region transfer json filepath: %s\n", json_filepath); + LOG_DEBUG("Region transfer current state: %s\n", region_mapping->region_state.cur_state); + LOG_DEBUG("Region transfer store state: %s\n", region_mapping->region_state.store_state); + LOG_DEBUG("Region transfer client state: %s\n", region_mapping->region_state.client_state); + + cur_state_str = strdup(region_mapping->region_state.client_state); + client_state_str = region_mapping->region_state.client_state; + store_state_str = region_mapping->region_state.store_state; + } + else if (access_type == PDC_WRITE) + LOG_DEBUG("Region transfer does NOT have an attached graph\n"); + + if (access_type == PDC_WRITE) { + MEMCPY_INC(json_filepath ? json_filepath : "", strlen(json_filepath ? json_filepath : "") + 1); + MEMCPY_INC(cur_state_str ? cur_state_str : "", strlen(cur_state_str ? cur_state_str : "") + 1); + MEMCPY_INC(client_state_str ? client_state_str : "", + strlen(client_state_str ? client_state_str : "") + 1); + MEMCPY_INC(store_state_str ? store_state_str : "", + strlen(store_state_str ? store_state_str : "") + 1); + } } for (i = 0; i < n_objs; ++i) { @@ -1271,6 +1390,7 @@ PDC_Client_pack_all_requests(int n_objs, pdc_transfer_request_start_all_pkg **tr sizeof(uint64_t) * transfer_requests[i]->transfer_request->remote_region_ndim); MEMCPY_INC(transfer_requests[i]->transfer_request->obj_dims, sizeof(uint64_t) * transfer_requests[i]->transfer_request->obj_ndim); + // Note buf is undefined for PDC_READ if (access_type == PDC_WRITE) { MEMCPY_INC(transfer_requests[i]->buf, obj_data_size); @@ -1544,32 +1664,31 @@ PDCregion_transfer_start_all_mpi(pdcid_t *transfer_request_id, int size, MPI_Com } #endif -perr_t -PDCregion_transfer_start_common(pdcid_t transfer_request_id, +static perr_t +region_transfer_start_common_helper(pdcid_t transfer_request_id, #ifdef ENABLE_MPI - MPI_Comm comm) + MPI_Comm comm) #else - int comm) + int comm) #endif { FUNC_ENTER(NULL); perr_t ret_value = SUCCEED; - struct _pdc_id_info * transferinfo; + struct _pdc_id_info * transfer_info; pdc_transfer_request *transfer_request; size_t unit; int i; hg_bulk_t bulk_handle; - if ((transferinfo = PDC_find_id(transfer_request_id)) == NULL) - PGOTO_DONE(ret_value); - - transfer_request = (pdc_transfer_request *)(transferinfo->obj_ptr); + if ((transfer_info = PDC_find_id(transfer_request_id)) == NULL) + PGOTO_ERROR(FAIL, "Failed to find id: %d", transfer_request_id); + transfer_request = (pdc_transfer_request *)(transfer_info->obj_ptr); if (transfer_request->metadata_id != NULL) PGOTO_ERROR(FAIL, "PDC_Client attempted to start existing transfer request"); - // Dynamic case is implemented within the the aggregated version. The main reason is that the target data + // Dynamic case is implemented within the aggregated version. The main reason is that the target data // server may not be unique, so we may end up sending multiple requests to the same data server. // Aggregated method will take care of this type of operation. if (transfer_request->region_partition == PDC_REGION_DYNAMIC || @@ -1624,7 +1743,7 @@ PDCregion_transfer_start_common(pdcid_t transfer_request_id, transfer_request->obj_servers[i], transfer_request->obj_ndim, transfer_request->obj_dims, transfer_request->remote_region_ndim, transfer_request->output_offsets[i], transfer_request->output_sizes[i], unit, transfer_request->access_type, - transfer_request->metadata_id + i); + transfer_request->metadata_id + i, transfer_request->obj_pointer); PDCregion_transfer_add_bulk_handle(transfer_request, bulk_handle); } } @@ -1645,19 +1764,30 @@ PDCregion_transfer_start_common(pdcid_t transfer_request_id, transfer_request->data_server_id, transfer_request->obj_ndim, transfer_request->obj_dims, transfer_request->remote_region_ndim, transfer_request->remote_region_offset, transfer_request->remote_region_size, unit, transfer_request->access_type, - transfer_request->metadata_id); + transfer_request->metadata_id, transfer_request->obj_pointer); PDCregion_transfer_add_bulk_handle(transfer_request, bulk_handle); } // For POSIX consistency, we block here until the data is received by the server - if (transfer_request->consistency == PDC_CONSISTENCY_POSIX) { + if (transfer_request->consistency == PDC_CONSISTENCY_POSIX) PDCregion_transfer_wait(transfer_request_id); - } done: FUNC_LEAVE(ret_value); } +perr_t +PDCregion_transfer_start_common(pdcid_t transfer_request_id, +#ifdef ENABLE_MPI + MPI_Comm comm) +#else + int comm) +#endif +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(region_transfer_start_common_helper(transfer_request_id, comm)); +} + perr_t PDCregion_transfer_start(pdcid_t transfer_request_id) { @@ -1741,7 +1871,6 @@ PDCregion_transfer_status(pdcid_t transfer_request_id, pdc_transfer_status_t *co *completed = PDC_TRANSFER_STATUS_COMPLETE; PGOTO_DONE(ret_value); } - transfer_request = (pdc_transfer_request *)(transferinfo->obj_ptr); if (transfer_request->metadata_id != NULL) { unit = transfer_request->unit; @@ -1771,6 +1900,7 @@ PDCregion_transfer_status(pdcid_t transfer_request_id, pdc_transfer_status_t *co transfer_request->output_offsets[i] = (uint64_t *)PDC_free(transfer_request->output_offsets[i]); } + // Copy read data from a contiguous buffer back to the user buffer using local data information. release_region_buffer( transfer_request->buf, transfer_request->obj_dims, transfer_request->local_region_ndim, @@ -2084,7 +2214,6 @@ PDCregion_transfer_wait(pdcid_t transfer_request_id) unit = transfer_request->unit; if (transfer_request->region_partition == PDC_REGION_STATIC) { - for (i = 0; i < transfer_request->n_obj_servers; ++i) { ret_value = PDC_Client_transfer_request_wait(transfer_request->metadata_id[i], transfer_request->obj_servers[i], diff --git a/src/api/pdc_tf/include/pdc_tf.h b/src/api/pdc_tf/include/pdc_tf.h new file mode 100644 index 000000000..a8ffb9056 --- /dev/null +++ b/src/api/pdc_tf/include/pdc_tf.h @@ -0,0 +1,64 @@ +#ifndef PDC_TF_H +#define PDC_TF_H + +/** + * List of FIXME:'s + * 1. Keep track of region history + * 2. Parse func name correctly when adding function + */ +#include "pdc_public.h" +#include "pdc_dg.h" +#include "pdc_tf_common.h" + +#ifndef TF_GRAPHS_DIR +#error "TF_GRAPHS_DIR was undefined" +#endif + +/** + * - PDC_TF_OVERWRITE: the transformation modifies the existing object. + * - PDC_TF_CREATE: the transformation creates a new object for the output. + */ +typedef enum pdc_tf_output_mode_t { + PDC_TF_OVERWRITE, + PDC_TF_CREATE, +} pdc_tf_output_mode_t; + +pdc_dg_t *PDCtf_get_dg(pdcid_t dg_id); + +/** + * specifies how the output of a transformation function should be handled + * the default is PDC_TF_OVERWRITE if this function is not called + */ +perr_t PDCtf_set_output_mode(pdcid_t dg_id, pdc_tf_output_mode_t mode, pdcid_t *obj_ids, char **result_names, + int num_ids); + +/** + * free resources used by directed graph + * this includes resources managed by functions/data states + */ +perr_t PDCtf_close_dg(pdcid_t dg_id); + +// region transfer to/from the specified obj_id, global_reg_id follow DG +perr_t PDCtf_attach_to_region(pdcid_t dg_id, pdcid_t obj_id, pdcid_t remote_reg, char *client_state, + char *store_state); + +// all region transfers for obj_id follow DG +perr_t PDCtf_attach_to_obj(pdcid_t dg_id, pdcid_t obj_id, char *client_state, char *store_state); + +// all region transfers for specified obj_ids follow DG +perr_t PDCtf_attach_to_objs(pdcid_t dg_id, pdcid_t *obj_ids, int num_ids, char *client_state, + char *store_state); + +// register data states, transformations, and directed graphs as types +perr_t PDCtf_init(); + +// print graph in readable format +void PDCtf_print_dg(pdcid_t dg_id, bool write_to_file); + +// print execution path +void PDCtf_print_exec_path(pdcid_t dg_id, char *cur_state, char *desired_state); + +// load JSON file describing directed graph +pdcid_t PDCtf_dg_json_create(char *json_filepath); + +#endif /* PDC_TF_H */ diff --git a/src/api/pdc_tf/pdc_tf.c b/src/api/pdc_tf/pdc_tf.c new file mode 100644 index 000000000..701ed6d2e --- /dev/null +++ b/src/api/pdc_tf/pdc_tf.c @@ -0,0 +1,252 @@ +#include +#include +#include +#include + +#include "pdc_tf.h" +#include "pdc_timing.h" +#include "pdc_interface.h" +#include "pdc_prop.h" +#include "pdc_obj_pkg.h" +#include "pdc_dg.h" +#include "pdc_malloc.h" +#include "pdc_region.h" +#include "pdc_tf_common.h" +#include "pdc_client_server_common.h" + +pdc_dg_t * +PDCtf_get_dg(pdcid_t dg_id) +{ + FUNC_ENTER(NULL); + + pdc_dg_t *ret_value = NULL; + + struct _pdc_id_info *id_info = PDC_find_id(dg_id); + if (id_info == NULL) + PGOTO_ERROR(NULL, "Failed to find dg_id"); + + ret_value = (pdc_dg_t *)id_info->obj_ptr; + if (ret_value == NULL) + PGOTO_ERROR(NULL, "Failed to find dg"); + +done: + FUNC_LEAVE(ret_value); +} + +pdcid_t +PDCtf_dg_json_create(char *json_filepath) +{ + FUNC_ENTER(NULL); + pdcid_t ret_value = 0; + + pdc_dg_t *dg = PDCtf_dg_json_create_common(json_filepath); + if (dg == NULL) { + abort(); + PGOTO_ERROR(FAIL, "Error with PDCtf_open_dg_json_common"); + } + + ret_value = PDC_id_register(PDC_TF_DG, dg); + +done: + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_close_dg(pdcid_t dg_id) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + if (PDC_dec_ref(dg_id) < 0) + PGOTO_ERROR(FAIL, "Directed graph: problem of freeing ID"); + +done: + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_set_output_mode(pdcid_t dg_id, pdc_tf_output_mode_t mode, pdcid_t *obj_ids, char **result_names, + int num_ids) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + FUNC_LEAVE(ret_value); +} + +/** + * Gets pdc_tf_obj_t from obj_id + * If pdc_tf_obj_t is NULL then one is allocated + */ +static perr_t +locate_and_set_pdc_tf_obj_t(pdcid_t obj_id, struct _pdc_obj_info **pdc_obj_info, + struct pdc_tf_obj_t **pdc_tf_obj) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + // First locate object + const struct _pdc_id_info *obj_id_info = PDC_find_id(obj_id); + if (obj_id_info == NULL) + PGOTO_ERROR(FAIL, "Failed to find object using pdcid"); + *pdc_obj_info = obj_id_info->obj_ptr; + + // Validate partition strategy is supported with transformations + /*if ((*pdc_obj_info)->obj_pt->obj_prop_pub->region_partition != PDC_OBJ_STATIC) { + LOG_WARNING("Partition strategy not supported for transformations\n"); + PGOTO_ERROR(FAIL, "The following partitions strategies are supported: PDC_OBJ_STATIC"); + }*/ + + // Validate user has set the datatype on the object + if (PDC_get_var_type_size((*pdc_obj_info)->obj_pt->obj_prop_pub->type) == 0) { + LOG_ERROR("Invalid data type for object transformation\n"); + PGOTO_ERROR(FAIL, "Data type must be set on object before attaching transformations"); + } + + // Pull out pdc obj transform information and allocate first if NULL + if ((*pdc_obj_info)->pdc_tf_obj == NULL) + (*pdc_obj_info)->pdc_tf_obj = PDC_calloc(1, sizeof(struct pdc_tf_obj_t)); + *pdc_tf_obj = (*pdc_obj_info)->pdc_tf_obj; + +done: + FUNC_LEAVE(ret_value); +} + +// Region transfer to/from the specified obj_id, remote_reg_id follow DG +perr_t +PDCtf_attach_to_region(pdcid_t dg_id, pdcid_t obj_id, pdcid_t remote_reg, char *client_state, + char *store_state) +{ + FUNC_ENTER(NULL); + + LOG_DEBUG("PDCtf_attach_to_region was called\n"); + + perr_t ret_value = SUCCEED; + + struct pdc_tf_obj_t * pdc_tf_obj = NULL; + struct _pdc_obj_info *pdc_obj_info = NULL; + if (locate_and_set_pdc_tf_obj_t(obj_id, &pdc_obj_info, &pdc_tf_obj) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with locate_and_set_pdc_tf_obj_t"); + + // Get region information + struct _pdc_id_info *region_id_info = PDC_find_id(remote_reg); + if (region_id_info == NULL) + PGOTO_ERROR(FAIL, "Cannot locate remote region ID"); + struct pdc_region_info *region_info = region_id_info->obj_ptr; + + // Create region mapping vector if needed + if (pdc_tf_obj->region_mappings_vector == NULL) + pdc_tf_obj->region_mappings_vector = pdc_vector_create(8, 2.0); + + // Add region mapping + // FIXME: if this is called twice on the same region should it overwrite or error + // need to look at this on the server side as well... + pdc_tf_region_mapping_t *region_mapping = PDC_calloc(1, sizeof(pdc_tf_region_mapping_t)); + pdc_vector_add(pdc_tf_obj->region_mappings_vector, region_mapping); + + pdc_tf_region_t *conceptual_region = ®ion_mapping->conceptual_region; + uint64_t * conceptual_offset = region_mapping->conceptual_offset; + + // Copy region information into conceptual region + PDCtf_set_tf_region_t(conceptual_region, region_info->ndim, pdc_obj_info->obj_pt->obj_prop_pub->type, + region_info->size); + memcpy(conceptual_offset, region_info->offset, region_info->ndim * sizeof(uint64_t)); + + // FIXME: need to free these strings later + region_mapping->region_state.client_state = strdup(client_state); + region_mapping->region_state.cur_state = strdup(client_state); + region_mapping->region_state.store_state = strdup(store_state); + region_mapping->region_state.dg_id = dg_id; +done: + FUNC_LEAVE(ret_value); +} + +// all region transfers for obj_id follow DG +perr_t +PDCtf_attach_to_obj(pdcid_t dg_id, pdcid_t obj_id, char *client_state, char *store_state) +{ + FUNC_ENTER(NULL); + + LOG_DEBUG("PDCtf_attach_to_obj was called\n"); + + perr_t ret_value = SUCCEED; + + struct pdc_tf_obj_t * pdc_tf_obj = NULL; + struct _pdc_obj_info *pdc_obj_info = NULL; + if (locate_and_set_pdc_tf_obj_t(obj_id, &pdc_obj_info, &pdc_tf_obj) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with locate_and_set_pdc_tf_obj_t"); + + if (pdc_tf_obj->attach_to_all_regions) + LOG_WARNING("Graph was already attached to obj\n"); + pdc_tf_obj->attach_to_all_regions = true; + + pdc_tf_obj->all_regions_state.client_state = strdup(client_state); + pdc_tf_obj->all_regions_state.cur_state = strdup(client_state); + pdc_tf_obj->all_regions_state.store_state = strdup(store_state); + pdc_tf_obj->all_regions_state.dg_id = dg_id; + +done: + FUNC_LEAVE(ret_value); +} + +// all region transfers for specified obj_ids follow DG +perr_t +PDCtf_attach_to_objs(pdcid_t dg_id, pdcid_t *obj_ids, int num_ids, char *client_state, char *store_state) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + for (int i = 0; i < num_ids; i++) { + if (PDCtf_attach_to_obj(dg_id, obj_ids[i], client_state, store_state) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDCtf_attach_to_obj"); + } + +done: + FUNC_LEAVE(ret_value); +} + +static perr_t +PDCtf_dg_free(void *dg) +{ + FUNC_ENTER(NULL); + + LOG_INFO("PDCtf_dg_free called\n"); + + perr_t ret_value = SUCCEED; + PDCdg_destroy((pdc_dg_t *)dg); + + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_init() +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + if (PDC_register_type(PDC_TF_DG, (PDC_free_t)PDCtf_dg_free) < 0) + PGOTO_ERROR(FAIL, "Failed to register PDC_TF_DG type"); + if (PDCtf_init_builtin_funcs() != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDCtf_init_builtin_funcs"); + +done: + FUNC_LEAVE(ret_value); +} + +void +PDCtf_print_dg(pdcid_t dg_id, bool write_to_file) +{ + pdc_dg_t *dg = PDCtf_get_dg(dg_id); + assert(dg != NULL); + PDCtf_print_dg_common(dg, write_to_file); +} + +void +PDCtf_print_exec_path(pdcid_t dg_id, char *cur_state, char *desired_state) +{ + PDCtf_print_exec_path_common(PDCtf_get_dg(dg_id), cur_state, desired_state); +} \ No newline at end of file diff --git a/src/api/pdc_transform/include/pdc_transform.h b/src/api/pdc_transform/include/pdc_transform.h deleted file mode 100644 index 45b4b1cc4..000000000 --- a/src/api/pdc_transform/include/pdc_transform.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_TRANSFORM_SUPPORT_H -#define PDC_TRANSFORM_SUPPORT_H - -#include "pdc_public.h" - -/***************************/ -/* Library Public Typedefs */ -/***************************/ -typedef enum { - PDC_TESTING = 0, - PDC_FILE_IO = 1, - PDC_DATA_MAP = 2, - PDC_PRE_ANALYSIS = 4, - PDC_POST_ANALYSIS = 8 -} pdc_obj_transform_t; - -typedef enum { DECR_STATE = -100, INCR_STATE = 100, _STATIC_STATE = 0 } pdc_state_next_t; - -typedef enum { DATA_IN = 1, DATA_OUT = 2, DATA_RELOCATION = 4 } pdc_data_movement_t; - -/*********************/ -/* Public Prototypes */ -/*********************/ -/** - * Register a function to be invoked at a specified point during execution - * to transform the supplied data. - * - * \param func [IN] String containing the [libraryname:]function to be registered. - * (default library name = "libpdctransforms") - * \param obj_id [IN] PDC object id containing the input data. - * \param current_state [IN] State/Sequence ID to identify when the transform can take place. - * \param next_state [IN] State/Sequence ID after the transform is complete (should be +1 or -1). - * \param op_type [IN] An enumerated ID specifying an operation type that invokes the transform. - * \param when [IN] An enumerated ID specifying when/where a transform is invoked. - * (examples for data movement: DATA_OUT, DATA_IN) - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDCobj_transform_register(char *func, pdcid_t obj_id, int current_state, int next_state, - pdc_obj_transform_t op_type, pdc_data_movement_t when); - -/** - * Register a function to be invoked as a result of having mapped two regions. - * The specfied transform function is invoked as a result of data movement between src and dest. - * - * \param func [IN] String containing the [libraryname:]function to be registered. - * (default library name = "libpdctransforms") - * \param src_region_id [IN] PDC region id of the data mapping source. - * \param dest_region_id [IN] PDC region id of the data mapping destination (target). - * \param current_state [IN] State/Sequence ID to identify when the transform can take place. - * \param next_state [IN] State/Sequence ID after the transform is complete (should be +1 or -1). - * \param when [IN] An enumerated ID specifying when/where a transform is invoked. - * (examples for data movement: DATA_OUT, DATA_IN) - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDCbuf_map_transform_register(char *func, void *buf, pdcid_t src_region_id, pdcid_t dest_object_id, - pdcid_t dest_region_id, int current_state, int next_state, - pdc_data_movement_t when); - -#endif /* PDC_TRANSFORM_SUPPORT_H */ diff --git a/src/api/pdc_transform/include/pdc_transforms_common.h b/src/api/pdc_transform/include/pdc_transforms_common.h deleted file mode 100644 index 5cc1c4ca6..000000000 --- a/src/api/pdc_transform/include/pdc_transforms_common.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_TRANSFORMS_COMMON_H -#define PDC_TRANSFORMS_COMMON_H - -#include "pdc_public.h" -#include "pdc_timing.h" -#include "mercury.h" -#include "mercury_macros.h" -#include "mercury_proc_string.h" - -/***************************************/ -/* Library-private Function Prototypes */ -/***************************************/ -hg_id_t PDC_analysis_ftn_register(hg_class_t *hg_class); -hg_id_t PDC_transform_ftn_register(hg_class_t *hg_class); - -/************************************/ -/* Local Type and Struct Definition */ -/************************************/ -/* Define transform_ftn_in_t */ -typedef struct transform_ftn_in_t { - hg_const_string_t ftn_name; - hg_const_string_t loadpath; - pdcid_t object_id; - pdcid_t region_id; - int32_t client_index; - int32_t operation_type; /* When, e.g. during mapping */ - int32_t start_state; - int32_t next_state; - int8_t op_type; - int8_t when; -} transform_ftn_in_t; - -/* Define transform_ftn_out_t */ -typedef struct transform_ftn_out_t { - pdcid_t object_id; - pdcid_t region_id; - int32_t client_index; - int32_t ret; -} transform_ftn_out_t; - -/* Define hg_proc_transform_ftn_in_t */ -static HG_INLINE hg_return_t -hg_proc_transform_ftn_in_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - transform_ftn_in_t *struct_data = (transform_ftn_in_t *)data; - ret = hg_proc_hg_const_string_t(proc, &struct_data->ftn_name); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_hg_const_string_t(proc, &struct_data->loadpath); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->object_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->region_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->client_index); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->operation_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->start_state); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->next_state); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int8_t(proc, &struct_data->op_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int8_t(proc, &struct_data->when); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -/* Define hg_proc_transform_ftn_out_t */ -static HG_INLINE hg_return_t -hg_proc_transform_ftn_out_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - transform_ftn_out_t *struct_data = (transform_ftn_out_t *)data; - ret = hg_proc_uint64_t(proc, &struct_data->object_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->region_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->client_index); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->ret); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -#endif /* PDC_TRANSFORMS_COMMON_H */ diff --git a/src/api/pdc_transform/include/pdc_transforms_pkg.h b/src/api/pdc_transform/include/pdc_transforms_pkg.h deleted file mode 100644 index 4929f1f12..000000000 --- a/src/api/pdc_transform/include/pdc_transforms_pkg.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ -#ifndef PDC_TRANSFORMS_H -#define PDC_TRANSFORMS_H - -#include "pdc_private.h" -#include "pdc_transform.h" - -#define DATA_ANY 7 - -/***************************/ -/* Library Private Structs */ -/***************************/ -typedef enum { C_lang = 0, FORTRAN_lang, PYTHON_lang, JULIA_lang, N_LANGUAGES } _pdc_analysis_language_t; - -struct _pdc_region_transform_ftn_info { - pdcid_t object_id; - pdcid_t region_id; - int local_regIndex; - int meta_index; - struct pdc_region_info *src_region; - struct pdc_region_info *dest_region; - size_t (*ftnPtr)(); - int ftn_lastResult; - int readyState; - int nextState; - int client_id; - size_t type_extent; - size_t dest_extent; - pdc_var_type_t type; - pdc_var_type_t dest_type; - pdc_obj_transform_t op_type; - pdc_data_movement_t when; - _pdc_analysis_language_t lang; - void * data; - void * result; -}; - -/***************************************/ -/* Library-private Function Prototypes */ -/***************************************/ -/** - * To end PDC transform - * - * \return Non-negative on success/Negative on failure - */ -perr_t PDC_transform_end(); - -#endif /* PDC_TRANSFORMS_H */ diff --git a/src/api/pdc_transform/pdc_transform.c b/src/api/pdc_transform/pdc_transform.c deleted file mode 100644 index 111893380..000000000 --- a/src/api/pdc_transform/pdc_transform.c +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include "pdc_malloc.h" -#include "pdc_private.h" -#include "pdc_id_pkg.h" -#include "pdc_prop.h" -#include "pdc_obj.h" -#include "pdc_obj_pkg.h" -#include "pdc_region_pkg.h" -#include "pdc_region.h" -#include "pdc_interface.h" -#include "pdc_transform.h" -#include "pdc_analysis_pkg.h" -#include "pdc_client_server_common.h" -#include "pdc_utlist.h" -#include "pdc_logger.h" -#include -#include -#include -#include - -extern int pdc_client_mpi_rank_g; -extern int pdc_client_mpi_size_g; - -static char *default_pdc_transforms_lib = "libpdctransforms.so"; - -perr_t -PDCobj_transform_register(char *func, pdcid_t obj_id, int current_state, int next_state, - pdc_obj_transform_t op_type, pdc_data_movement_t when) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - void * ftnHandle = NULL; - size_t (*ftnPtr)() = NULL; - struct _pdc_region_transform_ftn_info *thisFtn = NULL; - struct _pdc_obj_info * obj1, *obj2; - struct _pdc_id_info * objinfo1; - struct _pdc_obj_prop * prop; - struct pdc_region_info * reg1 = NULL, *reg2 = NULL; - pdcid_t src_region_id = 0, dest_region_id = 0; - pdcid_t dest_object_id = 0; - char * thisApp = NULL; - char * colonsep = NULL; - char * transformslibrary = NULL; - char * applicationDir = NULL; - char * userdefinedftn = NULL; - char * loadpath = NULL; - int local_regIndex; - struct _pdc_id_info * id_info; - - thisApp = PDC_get_argv0_(); - if (thisApp) - applicationDir = dirname(strdup(thisApp)); - userdefinedftn = strdup(func); - - if ((colonsep = strrchr(userdefinedftn, ':')) != NULL) { - *colonsep++ = 0; - transformslibrary = colonsep; - } - else - transformslibrary = default_pdc_transforms_lib; - - // TODO: - // Should probably validate the location of the "transformslibrary" - // - loadpath = PDC_get_realpath(transformslibrary, applicationDir); - - if (PDC_get_ftnPtr_(userdefinedftn, loadpath, &ftnHandle) < 0) - PGOTO_ERROR(FAIL, "PDC_get_ftnPtr_ returned an error"); - - if ((ftnPtr = ftnHandle) == NULL) - PGOTO_ERROR(FAIL, "Transforms function lookup failed"); - - if ((thisFtn = (struct _pdc_region_transform_ftn_info *)PDC_malloc( - sizeof(struct _pdc_region_transform_ftn_info))) == NULL) - PGOTO_ERROR(FAIL, "PDC register_obj_transforms memory allocation failed"); - - memset(thisFtn, 0, sizeof(struct _pdc_region_transform_ftn_info)); - thisFtn->ftnPtr = (size_t(*)())ftnPtr; - thisFtn->object_id = obj_id; - thisFtn->op_type = op_type; - thisFtn->when = when; - thisFtn->lang = C_lang; - thisFtn->nextState = next_state; - thisFtn->dest_type = PDC_UNKNOWN; - - // Add to our own list of transform functions - if ((local_regIndex = PDC_add_transform_ptr_to_registry_(thisFtn)) < 0) - PGOTO_ERROR(FAIL, "PDC unable to register transform function"); - - // Flag the transform as being active on mapping operations - if (op_type == PDC_DATA_MAP) { - objinfo1 = PDC_find_id(obj_id); - if (objinfo1 == NULL) - PGOTO_ERROR(FAIL, "Cannot locate local object ID"); - obj1 = (struct _pdc_obj_info *)(objinfo1->obj_ptr); - /* See if any mapping operations are defined */ - if (obj1 && (obj1->region_list_head != NULL)) { - id_info = PDC_find_id(obj1->region_list_head->orig_reg_id); - src_region_id = obj1->region_list_head->orig_reg_id; - dest_region_id = obj1->region_list_head->des_reg_id; - // mapping is already defined... - if (id_info && ((reg1 = (struct pdc_region_info *)id_info->obj_ptr) != NULL)) { - thisFtn->src_region = reg1; - obj1 = reg1->obj; - - // Requires that the PDCprop_set_obj_buf function be used... - if (obj1 && ((prop = obj1->obj_pt) != NULL)) { - thisFtn->data = prop->buf; - thisFtn->type = prop->obj_prop_pub->type; - thisFtn->type_extent = prop->type_extent; - } - } - id_info = PDC_find_id(dest_region_id); - if (id_info && ((reg2 = (struct pdc_region_info *)id_info->obj_ptr) != NULL)) { - thisFtn->dest_region = reg2; - obj2 = reg2->obj; - dest_object_id = obj2->obj_info_pub->local_id; - if (obj2 && ((prop = obj2->obj_pt) != NULL)) { - thisFtn->result = prop->buf; - thisFtn->dest_type = prop->obj_prop_pub->type; - thisFtn->dest_extent = prop->type_extent; - } - } - // Flag the destination region with the transform - reg2->registered_op |= PDC_TRANSFORM; - } - PDC_Client_register_region_transform(userdefinedftn, loadpath, src_region_id, dest_region_id, - dest_object_id, current_state, thisFtn->nextState, - (int)PDC_DATA_MAP, (int)when, local_regIndex); - } - -done: - if (applicationDir) - applicationDir = (char *)PDC_free(applicationDir); - if (userdefinedftn) - userdefinedftn = (char *)PDC_free(userdefinedftn); - - FUNC_LEAVE(ret_value); -} - -perr_t -PDCbuf_map_transform_register(char *func, void *buf, pdcid_t src_region_id, pdcid_t dest_object_id, - pdcid_t dest_region_id, int current_state, int next_state, - pdc_data_movement_t when) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; /* Return value */ - void * ftnHandle = NULL; - size_t (*ftnPtr)() = NULL; - struct _pdc_obj_info * object1 = NULL; - struct _pdc_region_transform_ftn_info *thisFtn = NULL; - struct pdc_region_info * region_info = NULL; - struct _pdc_id_info * id_info; - char * thisApp = NULL; - char * colonsep = NULL; - char * transformslibrary = NULL; - char * applicationDir = NULL; - char * userdefinedftn = NULL; - char * loadpath = NULL; - int local_regIndex; - - thisApp = PDC_get_argv0_(); - if (thisApp) - applicationDir = dirname(strdup(thisApp)); - userdefinedftn = strdup(func); - - if ((colonsep = strrchr(userdefinedftn, ':')) != NULL) { - *colonsep++ = 0; - transformslibrary = colonsep; - } - else - transformslibrary = default_pdc_transforms_lib; - - // TODO: - // Should probably validate the location of the "transformslibrary" - // - loadpath = PDC_get_realpath(transformslibrary, applicationDir); - - if (PDC_get_ftnPtr_(userdefinedftn, loadpath, &ftnHandle) < 0) - PGOTO_ERROR(FAIL, "PDC_get_ftnPtr_ returned an error"); - - if ((ftnPtr = ftnHandle) == NULL) - PGOTO_ERROR(FAIL, "Transforms function lookup failed"); - - if ((thisFtn = (struct _pdc_region_transform_ftn_info *)PDC_malloc( - sizeof(struct _pdc_region_transform_ftn_info))) == NULL) - PGOTO_ERROR(FAIL, "PDC register_obj_transforms memory allocation failed"); - - thisFtn->ftnPtr = (size_t(*)())ftnPtr; - thisFtn->object_id = dest_object_id; - id_info = PDC_find_id(src_region_id); - if (id_info && ((region_info = (struct pdc_region_info *)id_info->obj_ptr) != NULL)) - thisFtn->src_region = region_info; - - id_info = PDC_find_id(dest_region_id); - if (id_info && ((region_info = (struct pdc_region_info *)id_info->obj_ptr) != NULL)) - thisFtn->dest_region = region_info; - - // Flag the destination region with the transform - // We do this here because the target region is what - // will eventually be locked and then unlocked to enable - // a mapping data transfer. - region_info->registered_op |= PDC_TRANSFORM; - - thisFtn->op_type = PDC_DATA_MAP; - thisFtn->when = when; - thisFtn->lang = C_lang; - thisFtn->client_id = pdc_client_mpi_rank_g; - thisFtn->readyState = current_state; - thisFtn->ftn_lastResult = 0; - thisFtn->data = buf; - id_info = PDC_find_id(dest_object_id); - if (id_info) - object1 = (struct _pdc_obj_info *)(id_info->obj_ptr); - if (object1) { - thisFtn->type = object1->obj_pt->obj_prop_pub->type; - if (object1->obj_pt->type_extent == 0) { - object1->obj_pt->type_extent = PDC_get_var_type_size(object1->obj_pt->obj_prop_pub->type); - } - thisFtn->type_extent = object1->obj_pt->type_extent; - thisFtn->dest_extent = object1->obj_pt->type_extent; - thisFtn->dest_type = object1->obj_pt->obj_prop_pub->type; - } - if (next_state == INCR_STATE) - thisFtn->nextState = current_state + 1; - else if (next_state == DECR_STATE) - thisFtn->nextState = current_state - 1; - else - thisFtn->nextState = current_state; - - // Add to our own list of transforms functions - if ((local_regIndex = PDC_add_transform_ptr_to_registry_(thisFtn)) < 0) - PGOTO_ERROR(FAIL, "PDC unable to register transform function"); - - PDC_Client_register_region_transform(userdefinedftn, loadpath, src_region_id, dest_region_id, - dest_object_id, current_state, thisFtn->nextState, (int)PDC_DATA_MAP, - (int)when, local_regIndex); - -done: - if (applicationDir) - applicationDir = (char *)PDC_free(applicationDir); - if (userdefinedftn) - userdefinedftn = (char *)PDC_free(userdefinedftn); - if (loadpath) - loadpath = (char *)PDC_free(loadpath); - - FUNC_LEAVE(ret_value); -} - -perr_t -PDCbuf_io_transform_register(char *func ATTRIBUTE(unused), void *buf ATTRIBUTE(unused), - pdcid_t src_region_id ATTRIBUTE(unused), int current_state ATTRIBUTE(unused), - int next_state ATTRIBUTE(unused), pdc_data_movement_t when ATTRIBUTE(unused)) -{ - FUNC_ENTER(NULL); - - LOG_ERROR("IO transforms are not currently supported!\n"); - - FUNC_LEAVE(FAIL); -} - -perr_t -PDC_transform_end() -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - PDC_free_transform_registry(); - - FUNC_LEAVE(ret_value); -} diff --git a/src/api/pdc_transform/pdc_transforms_common.c b/src/api/pdc_transform/pdc_transforms_common.c deleted file mode 100644 index c4bbbcc7c..000000000 --- a/src/api/pdc_transform/pdc_transforms_common.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -/************************************************************************ - * This file includes the functionality to support PDC transforms - ************************************************************************ */ - -#include "pdc_analysis_pkg.h" -#include "pdc_transforms_common.h" -#include "pdc_client_server_common.h" -#include "pdc_logger.h" - -#ifdef ENABLE_MULTITHREAD -extern hg_thread_pool_t *hg_test_thread_pool_g; -#endif - -// transform_ftn_cb(hg_handle_t handle) -HG_TEST_RPC_CB(transform_ftn, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - transform_ftn_in_t in; - transform_ftn_out_t out = {0, 0, 0, -1}; - struct _pdc_region_transform_ftn_info *thisFtn = NULL; - void * ftnHandle = NULL; - - HG_Get_input(handle, &in); - - if (PDC_get_ftnPtr_(in.ftn_name, in.loadpath, &ftnHandle) >= 0) { - thisFtn = PDC_malloc(sizeof(struct _pdc_region_transform_ftn_info)); - if (thisFtn == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "transform_ftn_cb: Memory allocation failed"); - /* This sets up the index return for the client! - * We probably need to add more info to the - * 'thisFtn' structure, but this should be a - * decent start. - * - * TODO: Add more info. - * Note that we (the server in this case) may not - * have the actual object or region. The mapping - * protocal is that the server gets a copy of the - * data once a WRITE-UNLOCK happens. At that point - * the server may get an indication that it has a - * transform to perform upon receipt of the data, - * e.g. uncompress. - * The client return contains the storage index of - * the transform; which it should cache as part of - * client data structures. This should allow it - * the client to include the cached index value - * as part of a future data movement request. - */ - thisFtn->ftnPtr = (size_t(*)())ftnHandle; - thisFtn->object_id = in.object_id; - thisFtn->region_id = in.region_id; - thisFtn->op_type = (pdc_obj_transform_t)in.op_type; - out.ret = PDC_add_transform_ptr_to_registry_(thisFtn); - out.client_index = in.client_index; - out.object_id = in.object_id; - out.region_id = in.region_id; - } - else { - LOG_ERROR("Unable to resolve transform function pointer\n"); - out.ret = -1; - } - -done: - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - - FUNC_LEAVE(ret_value); -} - -HG_TEST_THREAD_CB(transform_ftn) - -hg_id_t -PDC_transform_ftn_register(hg_class_t *hg_class) -{ - FUNC_ENTER(NULL); - - hg_id_t ret_value; - - ret_value = MERCURY_REGISTER(hg_class, "transform_ftn", transform_ftn_in_t, transform_ftn_out_t, - transform_ftn_cb); - - FUNC_LEAVE(ret_value); -} diff --git a/src/commons/collections/include/pdc_dg.h b/src/commons/collections/include/pdc_dg.h new file mode 100644 index 000000000..c74669abc --- /dev/null +++ b/src/commons/collections/include/pdc_dg.h @@ -0,0 +1,186 @@ +#ifndef PDC_DG_H +#define PDC_DG_H + +#include +#include +#include + +#define PDC_DG_INIT_EDGE_CAPACITY 4 +#define PDC_DG_INIT_VERTEX_CAPACITY 4 +#define PDC_DG_INVALID_EDGE -1 +#define PDC_DG_INVALID_VERTEX -1 + +/** + * Type for directed graph vertex identifiers. + */ +typedef int pdc_dg_vertex_id_t; + +/** + * Type for directed graph edge identifiers. + */ +typedef int pdc_dg_edge_id_t; + +/** + * Represents a vertex in the directed graph. + * + * \param vertex_id Unique identifier for the vertex. + * \param data Pointer to user-defined data associated with the vertex. + */ +typedef struct pdc_dg_vertex_t { + pdc_dg_vertex_id_t vertex_id; + void * data; +} pdc_dg_vertex_t; + +/** + * Represents an edge in the directed graph. + * + * \param edge_id Unique identifier for the edge. + * \param v1_id Source vertex ID. + * \param v2_id Destination vertex ID. + * \param data Pointer to user-defined data associated with the edge. + */ +typedef struct pdc_dg_edge_t { + pdc_dg_edge_id_t edge_id; + pdc_dg_vertex_id_t v1_id; + pdc_dg_vertex_id_t v2_id; + void * data; +} pdc_dg_edge_t; + +/** + * Represents a directed graph structure. + * + * All vertex and edge memory is managed internally. When the graph is destroyed, + * the user-defined `vertex_data_free` and `edge_data_free` functions (if provided) + * are called once for each vertex and edge respectively. The `dg_data_free` + * function is called once for the graph-level data. All other memory is freed + * by the library. + * + * \param edges Array of pointers to edges. + * \param vertices Array of pointers to vertices. + * \param edge_count Number of edges currently in the graph. + * \param vertex_count Number of vertices currently in the graph. + * \param vertex_capacity Maximum number of vertices before resizing. + * \param edge_capacity Maximum number of edges before resizing. + * \param data Pointer to user-defined graph-level data. + * \param vertices_are_equal Function to compare vertex data for equality. + * \param dg_data_free Function to free graph-level user data. + * \param edge_data_free Function to free edge-level user data. + * \param vertex_data_free Function to free vertex-level user data. + */ +typedef struct pdc_dg_t { + pdc_dg_edge_t ** edges; + pdc_dg_vertex_t **vertices; + + uint32_t edge_count; + uint32_t vertex_count; + + uint32_t vertex_capacity; + uint32_t edge_capacity; + + void *data; + + bool (*vertices_are_equal)(void *v1_data, void *v2_data); + void (*dg_data_free)(void *data); + void (*edge_data_free)(void *data); + void (*vertex_data_free)(void *data); +} pdc_dg_t; + +/** + * Create a new directed graph. + * + * Passing NULL for any of the free function pointers is allowed if the user + * intends to manage the memory manually. However, the vertices_are_equal + * function must be provided. + * + * \param data User data to associate with the graph. + * \param vertices_are_equal Function to compare two vertex data pointers. + * \param dg_data_free Function to free graph-level data (or NULL). + * \param edge_data_free Function to free edge-level data (or NULL). + * \param vertex_data_free Function to free vertex-level data (or NULL). + * + * \return Pointer to the new graph on success, NULL on failure. + */ +pdc_dg_t *PDCdg_create(void *data, bool (*vertices_are_equal)(void *v1_data, void *v2_data), + void (*dg_data_free)(void *data), void (*edge_data_free)(void *data), + void (*vertex_data_free)(void *data)); + +/** + * Destroy a directed graph and free all associated memory. + * + * This function will: + * - Free all edge data using `edge_data_free`, if provided. + * - Free all vertex data using `vertex_data_free`, if provided. + * - Free the graph-level user data using `dg_data_free`, if provided. + * - Deallocate all internal memory used by the graph. + * + * \param dg Pointer to the graph to destroy. + */ +void PDCdg_destroy(pdc_dg_t *dg); + +/** + * Add a vertex to the graph. + * + * \param dg Pointer to the graph. + * \param data User data to associate with the vertex. + * + * \return ID of the new vertex, or PDC_DG_INVALID_VERTEX on failure. + */ +pdc_dg_vertex_id_t PDCdg_add_vertex(pdc_dg_t *dg, void *data); + +/** + * Add a directed edge between two vertices. + * + * \param dg Pointer to the graph. + * \param v1_data Data of the source vertex. + * \param v2_data Data of the destination vertex. + * \param edge_data User data to associate with the edge. + * + * \return ID of the new edge, or PDC_DG_INVALID_EDGE on failure. + */ +pdc_dg_edge_id_t PDCdg_add_edge(pdc_dg_t *dg, void *v1_data, void *v2_data, void *edge_data); + +/** + * Check whether a vertex with the given data exists in the graph. + * + * \param dg Pointer to the graph. + * \param vertex_data Vertex data to check. + * + * \return ID of the vertex if it exists, or PDC_DG_INVALID_VERTEX. + */ +pdc_dg_vertex_id_t PDCdg_vertex_exists(pdc_dg_t *dg, void *vertex_data); + +/** + * Retrieve data associated with vertex + * + * \param dg Pointer to the graph. + * \param vertex_id Vertex id with data. + * + * \return Data of the vertex if it exists, or NULL. + */ +void *PDCdg_get_vertex_data(pdc_dg_t *dg, pdc_dg_vertex_id_t vertex_id); + +/** + * Retrieve data associated with edge + * + * \param dg Pointer to the graph. + * \param edge_id Edge id with data. + * + * \return Data of the edge if it exists, or NULL. + */ +void *PDCdg_get_edge_data(pdc_dg_t *dg, pdc_dg_edge_id_t edge_id); + +/** + * Find the shortest path between two vertices. + * + * \param dg Pointer to the graph. + * \param v1_data Data for the source vertex. + * \param v2_data Data for the destination vertex. + * \param edges_out Output array of edge pointers forming the path. + * \param num_edges Output number of edges in the path. + * + * \return true if a path is found, false otherwise. + */ +bool PDCdg_shortest_path(pdc_dg_t *dg, void *v1_data, void *v2_data, pdc_dg_edge_t **edges_out, + uint32_t *num_edges); + +#endif /* PDC_DG_H */ diff --git a/src/commons/collections/pdc_dg.c b/src/commons/collections/pdc_dg.c new file mode 100644 index 000000000..53a6db7b5 --- /dev/null +++ b/src/commons/collections/pdc_dg.c @@ -0,0 +1,425 @@ +#include "pdc_dg.h" +#include +#include +#include + +static bool +resize_dg(pdc_dg_t *dg, uint32_t new_vertex_count, uint32_t new_edge_count) +{ + bool ret_value = true; + + if (dg == NULL) { + ret_value = false; + goto done; + } + + // Resize vertices if needed + if (dg->vertex_capacity == 0) + dg->vertex_capacity = 1; + if (dg->vertex_capacity < new_vertex_count) { + while (dg->vertex_capacity < new_vertex_count) + dg->vertex_capacity *= 2; + + dg->vertices = + (pdc_dg_vertex_t **)realloc(dg->vertices, sizeof(pdc_dg_vertex_t *) * dg->vertex_capacity); + } + // Resize edges if needed + if (dg->edge_capacity == 0) + dg->edge_capacity = 1; + if (dg->edge_capacity < new_edge_count) { + while (dg->edge_capacity < new_edge_count) + dg->edge_capacity *= 2; + dg->edges = (pdc_dg_edge_t **)realloc(dg->edges, sizeof(pdc_dg_edge_t *) * dg->edge_capacity); + } + +done: + return ret_value; +} + +pdc_dg_t * +PDCdg_create(void *data, bool (*vertices_are_equal)(void *v1_data, void *v2_data), + void (*dg_data_free)(void *data), void (*edge_data_free)(void *data), + void (*vertex_data_free)(void *data)) +{ + pdc_dg_t *ret_value = NULL; + + if (vertices_are_equal == NULL) { + printf("vertices_are_equal function is required\n"); + goto done; + } + + ret_value = (pdc_dg_t *)calloc(1, sizeof(pdc_dg_t)); + + ret_value->vertex_capacity = PDC_DG_INIT_VERTEX_CAPACITY; + ret_value->edge_capacity = PDC_DG_INIT_EDGE_CAPACITY; + ret_value->vertices = (pdc_dg_vertex_t **)calloc(ret_value->vertex_capacity, sizeof(pdc_dg_vertex_t *)); + ret_value->edges = (pdc_dg_edge_t **)calloc(ret_value->edge_capacity, sizeof(pdc_dg_edge_t *)); + + ret_value->vertex_count = 0; + ret_value->edge_count = 0; + ret_value->data = data; + + ret_value->dg_data_free = dg_data_free; + ret_value->edge_data_free = edge_data_free; + ret_value->vertex_data_free = vertex_data_free; + ret_value->vertices_are_equal = vertices_are_equal; + +done: + return ret_value; +} + +void +PDCdg_destroy(pdc_dg_t *dg) +{ + if (dg == NULL) { + printf("dg was NULL\n"); + return; + } + + // first check that there are edges + if (dg->edges) { + for (int i = 0; i < dg->edge_count; i++) { + /** + * if user has defined a edge_data_free function + * call it on each edge's data + */ + if (dg->edges[i] && dg->edges[i]->data && dg->edge_data_free) + dg->edge_data_free(dg->edges[i]->data); + // free the edge + if (dg->edges[i]) + free(dg->edges[i]); + } + free(dg->edges); + } + // first check that there are vertices + if (dg->vertices) { + for (int i = 0; i < dg->vertex_count; i++) { + /** + * if user has defined a vertex_data_free function + * call it on each vertex's data + */ + if (dg->vertices[i] && dg->vertices[i]->data && dg->vertex_data_free) + dg->vertex_data_free(dg->vertices[i]->data); + // free the vertex + if (dg->vertices[i]) + free(dg->vertices[i]); + } + free(dg->vertices); + } + + /** + * if user has defined a dg_data_free function + * call it on dg's data + */ + if (dg->dg_data_free && dg->data) + dg->dg_data_free(dg->data); + + free(dg); +} + +pdc_dg_vertex_id_t +PDCdg_add_vertex(pdc_dg_t *dg, void *data) +{ + pdc_dg_vertex_id_t ret_value; + + if (dg == NULL) { + printf("dg was NULL\n"); + ret_value = PDC_DG_INVALID_VERTEX; + goto done; + } + + if (resize_dg(dg, dg->vertex_count + 1, dg->edge_count) != true) { + printf("Failed to resize dg\n"); + ret_value = PDC_DG_INVALID_VERTEX; + goto done; + } + + ret_value = dg->vertex_count; + dg->vertices[ret_value] = (pdc_dg_vertex_t *)calloc(1, sizeof(pdc_dg_vertex_t)); + dg->vertices[ret_value]->data = data; + dg->vertices[ret_value]->vertex_id = ret_value; + + dg->vertex_count++; + +done: + return ret_value; +} + +pdc_dg_edge_id_t +PDCdg_add_edge(pdc_dg_t *dg, void *v1_data, void *v2_data, void *edge_data) +{ + pdc_dg_edge_id_t ret_value; + + if (dg == NULL) { + printf("dg was NULL\n"); + return PDC_DG_INVALID_EDGE; + } + if (v1_data == NULL) { + printf("v1_data was NULL\n"); + return PDC_DG_INVALID_EDGE; + } + if (v2_data == NULL) { + printf("v2_data was NULL\n"); + return PDC_DG_INVALID_EDGE; + } + if (dg->vertices_are_equal(v1_data, v2_data)) { + printf("Vertices of edge were not unique\n"); + return PDC_DG_INVALID_EDGE; + } + if (resize_dg(dg, dg->vertex_count, dg->edge_count + 1) != true) { + printf("Failed to resize dg\n"); + return PDC_DG_INVALID_EDGE; + } + + ret_value = dg->edge_count; + dg->edges[ret_value] = (pdc_dg_edge_t *)calloc(1, sizeof(pdc_dg_edge_t)); + dg->edges[ret_value]->data = edge_data; + dg->edges[ret_value]->edge_id = ret_value; + + pdc_dg_vertex_id_t v1_id, v2_id = PDC_DG_INVALID_VERTEX; + + if ((v1_id = PDCdg_vertex_exists(dg, v1_data)) == PDC_DG_INVALID_VERTEX) + v1_id = PDCdg_add_vertex(dg, v1_data); + if ((v2_id = PDCdg_vertex_exists(dg, v2_data)) == PDC_DG_INVALID_VERTEX) + v2_id = PDCdg_add_vertex(dg, v2_data); + + dg->edges[ret_value]->v1_id = v1_id; + dg->edges[ret_value]->v2_id = v2_id; + + dg->edge_count++; + + return ret_value; +} + +pdc_dg_vertex_id_t +PDCdg_vertex_exists(pdc_dg_t *dg, void *vertex_data) +{ + if (dg == NULL) { + printf("pdc_dg_has_vertex called with NULL dg\n"); + return PDC_DG_INVALID_VERTEX; + } + + for (int i = 0; i < dg->vertex_count; i++) { + if (dg->vertices_are_equal(dg->vertices[i]->data, vertex_data)) + return dg->vertices[i]->vertex_id; + } + + return PDC_DG_INVALID_VERTEX; +} + +bool +PDCdg_shortest_path(pdc_dg_t *dg, void *v1_data, void *v2_data, pdc_dg_edge_t **edges_out, + uint32_t *num_edges) +{ + bool ret_value = false; + bool * visited = NULL; + pdc_dg_vertex_id_t *prev = NULL; + pdc_dg_vertex_id_t *queue = NULL; + pdc_dg_vertex_id_t *path = NULL; + *edges_out = NULL; + *num_edges = 0; + + if (dg == NULL) { + printf("dg was NULL\n"); + goto done; + } + + // Look up vertex IDs + pdc_dg_vertex_id_t from_vertex_id = PDCdg_vertex_exists(dg, v1_data); + pdc_dg_vertex_id_t to_vertex_id = PDCdg_vertex_exists(dg, v2_data); + + // Validate vertices + if (from_vertex_id >= dg->vertex_count) { + printf("Source vertex ID %u out of range (vertex_count=%u)\n", from_vertex_id, dg->vertex_count); + goto done; + } + if (to_vertex_id >= dg->vertex_count) { + printf("Destination vertex ID %u out of range (vertex_count=%u)\n", to_vertex_id, dg->vertex_count); + goto done; + } + if (from_vertex_id == PDC_DG_INVALID_VERTEX) { + printf("Source vertex not found\n"); + goto done; + } + if (to_vertex_id == PDC_DG_INVALID_VERTEX) { + printf("Destination vertex not found\n"); + goto done; + } + + uint32_t vertex_count = dg->vertex_count; + + // Allocate BFS data structures + visited = (bool *)calloc(vertex_count, sizeof(bool)); + prev = (pdc_dg_vertex_id_t *)malloc(vertex_count * sizeof(pdc_dg_vertex_id_t)); + queue = (pdc_dg_vertex_id_t *)malloc(vertex_count * sizeof(pdc_dg_vertex_id_t)); + + if (!visited || !prev || !queue) { + printf("Failed to allocate BFS structures\n"); + goto done; + } + + for (uint32_t i = 0; i < vertex_count; i++) + prev[i] = PDC_DG_INVALID_VERTEX; + + // Breadth-first search + uint32_t front = 0, rear = 0; + visited[from_vertex_id] = true; + queue[rear++] = from_vertex_id; + + bool found = false; + while (front < rear) { + pdc_dg_vertex_id_t current = queue[front++]; + + if (current == to_vertex_id) { + found = true; + break; + } + + for (uint32_t i = 0; i < dg->edge_count; i++) { + if (dg->edges[i]->v1_id == current) { + pdc_dg_vertex_id_t neighbor = dg->edges[i]->v2_id; + if (!visited[neighbor]) { + visited[neighbor] = true; + prev[neighbor] = current; + queue[rear++] = neighbor; + } + } + } + } + + if (!found) { + printf("No path found from vertex %d to %d\n", from_vertex_id, to_vertex_id); + goto done; + } + + // Reconstruct the path in reverse + uint32_t path_len = 0; + for (pdc_dg_vertex_id_t at = to_vertex_id; at != PDC_DG_INVALID_VERTEX; at = prev[at]) + path_len++; + + if (path_len < 2) { + printf("Path is too short to contain any edges\n"); + goto done; + } + + // Recover path vertices + path = (pdc_dg_vertex_id_t *)malloc(path_len * sizeof(pdc_dg_vertex_id_t)); + if (!path) { + printf("Failed to allocate path array\n"); + goto done; + } + + pdc_dg_vertex_id_t at = to_vertex_id; + for (int i = path_len - 1; i >= 0; i--) { + path[i] = at; + at = prev[at]; + } + + // Step 1: Count total edges along the path + uint32_t total_edges = 0; + for (uint32_t i = 0; i < path_len - 1; i++) { + pdc_dg_vertex_id_t from = path[i]; + pdc_dg_vertex_id_t to = path[i + 1]; + for (uint32_t j = 0; j < dg->edge_count; j++) { + if (dg->edges[j]->v1_id == from && dg->edges[j]->v2_id == to) { + total_edges++; + } + } + } + + if (total_edges == 0) { + printf("No edges found along path\n"); + goto done; + } + + // Step 2: Allocate edges_out + *edges_out = (pdc_dg_edge_t *)malloc(total_edges * sizeof(pdc_dg_edge_t)); + if (!*edges_out) { + printf("Failed to allocate edges_out\n"); + goto done; + } + *num_edges = total_edges; + + // Step 3: Copy all edges + uint32_t edge_idx = 0; + for (uint32_t i = 0; i < path_len - 1; i++) { + pdc_dg_vertex_id_t from = path[i]; + pdc_dg_vertex_id_t to = path[i + 1]; + + for (uint32_t j = 0; j < dg->edge_count; j++) { + if (dg->edges[j]->v1_id == from && dg->edges[j]->v2_id == to) { + memcpy(&(*edges_out)[edge_idx++], dg->edges[j], sizeof(pdc_dg_edge_t)); + } + } + } + + ret_value = true; + +done: + if (visited) + free(visited); + if (prev) + free(prev); + if (queue) + free(queue); + if (path) + free(path); + + return ret_value; +} + +/** + * Retrieve data associated with vertex + * + * \param dg Pointer to the graph. + * \param vertex_id Vertex id with data. + * + * \return Data of the vertex if it exists, or NULL. + */ +void * +PDCdg_get_vertex_data(pdc_dg_t *dg, pdc_dg_vertex_id_t vertex_id) +{ + void *ret_value = NULL; + + if (dg == NULL) { + printf("dg was NULL\n"); + goto done; + } + if (dg->vertices == NULL) { + printf("dg->vertices was NULL\n"); + goto done; + } + + ret_value = dg->vertices[vertex_id]->data; + +done: + return ret_value; +} + +/** + * Retrieve data associated with edge + * + * \param dg Pointer to the graph. + * \param edge_id Edge id with data. + * + * \return Data of the edge if it exists, or NULL. + */ +void * +PDCdg_get_edge_data(pdc_dg_t *dg, pdc_dg_edge_id_t edge_id) +{ + void *ret_value = NULL; + + if (dg == NULL) { + printf("dg was NULL\n"); + goto done; + } + if (dg->edges == NULL) { + printf("dg->edges was NULL\n"); + goto done; + } + + ret_value = dg->edges[edge_id]->data; + +done: + return dg->edges[edge_id]; +} \ No newline at end of file diff --git a/src/commons/collections/pdc_vector.c b/src/commons/collections/pdc_vector.c index 3a5285c15..a42f5fad2 100644 --- a/src/commons/collections/pdc_vector.c +++ b/src/commons/collections/pdc_vector.c @@ -1,30 +1,25 @@ #include "pdc_vector.h" -#include "pdc_timing.h" -#include "pdc_malloc.h" PDC_VECTOR * pdc_vector_new() { - FUNC_ENTER(NULL); - FUNC_LEAVE(pdc_vector_create(100, 2.0)); + return pdc_vector_create(100, 2.0); } PDC_VECTOR * pdc_vector_create(size_t initial_capacity, double expansion_factor) { - FUNC_ENTER(NULL); - // Allocate memory for the vector struct. - PDC_VECTOR *vector = (PDC_VECTOR *)PDC_malloc(sizeof(PDC_VECTOR)); + PDC_VECTOR *vector = (PDC_VECTOR *)malloc(sizeof(PDC_VECTOR)); if (vector == NULL) { - FUNC_LEAVE(NULL); + return NULL; } // Allocate memory for the array of items. - vector->items = (void **)PDC_malloc(initial_capacity * sizeof(void *)); + vector->items = (void **)malloc(initial_capacity * sizeof(void *)); if (vector->items == NULL) { - vector = (PDC_VECTOR *)PDC_free(vector); - FUNC_LEAVE(NULL); + free(vector); + return NULL; } // Initialize the vector fields. @@ -32,164 +27,138 @@ pdc_vector_create(size_t initial_capacity, double expansion_factor) vector->capacity = initial_capacity; vector->expansion_factor = expansion_factor; - FUNC_LEAVE(vector); + return vector; } void pdc_vector_destroy(PDC_VECTOR *vector) { - FUNC_ENTER(NULL); - if (vector == NULL) { - FUNC_LEAVE_VOID(); + return; } // Free all allocated memory for each item. for (size_t i = 0; i < vector->item_count; i++) { - vector->items[i] = (void *)PDC_free(vector->items[i]); + free(vector->items[i]); } // Free the array of items and the vector struct. - vector->items = (void **)PDC_free(vector->items); - vector = (PDC_VECTOR *)PDC_free(vector); - - FUNC_LEAVE_VOID(); + free(vector->items); + free(vector); } void pdc_vector_add(PDC_VECTOR *vector, void *item) { - FUNC_ENTER(NULL); - if (vector == NULL || item == NULL) { - FUNC_LEAVE_VOID(); + return; } // Expand the array of items if necessary. if (vector->item_count >= vector->capacity) { vector->capacity *= vector->expansion_factor; - vector->items = (void **)PDC_realloc(vector->items, vector->capacity * sizeof(void *)); + vector->items = (void **)realloc(vector->items, vector->capacity * sizeof(void *)); if (vector->items == NULL) { - FUNC_LEAVE_VOID(); + return; } } // Add the new item to the end of the array. vector->items[vector->item_count++] = item; - - FUNC_LEAVE_VOID(); } void * pdc_vector_get(PDC_VECTOR *vector, size_t index) { - FUNC_ENTER(NULL); - if (vector == NULL || index >= vector->item_count) { - FUNC_LEAVE(NULL); + return NULL; } // Return a pointer to the item at the given index. - FUNC_LEAVE(vector->items[index]); + return vector->items[index]; } size_t pdc_vector_size(PDC_VECTOR *vector) { - FUNC_ENTER(NULL); - if (vector == NULL) { - FUNC_LEAVE(0); + return 0; } // Return the number of items in the vector. - FUNC_LEAVE(vector->item_count); + return vector->item_count; } void pdc_vector_set_expansion_factor(PDC_VECTOR *vector, double expansion_factor) { - FUNC_ENTER(NULL); - if (vector == NULL) { - FUNC_LEAVE_VOID(); + return; } // Set the new expansion factor for the vector. vector->expansion_factor = expansion_factor; - - FUNC_LEAVE_VOID(); } double pdc_vector_get_expansion_factor(PDC_VECTOR *vector) { - FUNC_ENTER(NULL); - if (vector == NULL) { - FUNC_LEAVE(0); + return 0; } // Return the current expansion factor for the vector. - FUNC_LEAVE(vector->expansion_factor); + return vector->expansion_factor; } PDC_VECTOR_ITERATOR * pdc_vector_iterator_new(PDC_VECTOR *vector) { - FUNC_ENTER(NULL); - if (vector == NULL) { - FUNC_LEAVE(NULL); + return NULL; } // Allocate memory for the iterator struct. - PDC_VECTOR_ITERATOR *iterator = (PDC_VECTOR_ITERATOR *)PDC_malloc(sizeof(PDC_VECTOR_ITERATOR)); + PDC_VECTOR_ITERATOR *iterator = (PDC_VECTOR_ITERATOR *)malloc(sizeof(PDC_VECTOR_ITERATOR)); if (iterator == NULL) { - FUNC_LEAVE(NULL); + return NULL; } // Initialize the iterator fields. iterator->vector = vector; iterator->index = 0; - FUNC_LEAVE(iterator); + return iterator; } void pdc_vector_iterator_destroy(PDC_VECTOR_ITERATOR *iterator) { if (iterator == NULL) - FUNC_LEAVE_VOID(); + return; // Free the iterator struct. - iterator = (PDC_VECTOR_ITERATOR *)PDC_free(iterator); - - FUNC_LEAVE_VOID(); + free(iterator); } void * pdc_vector_iterator_next(PDC_VECTOR_ITERATOR *iterator) { - FUNC_ENTER(NULL); - if (iterator == NULL) { - FUNC_LEAVE(NULL); + return NULL; } // Return the next item in the vector. - FUNC_LEAVE(pdc_vector_get(iterator->vector, iterator->index++)); + return pdc_vector_get(iterator->vector, iterator->index++); } int pdc_vector_iterator_has_next(PDC_VECTOR_ITERATOR *iterator) { - FUNC_ENTER(NULL); - if (iterator == NULL) { - FUNC_LEAVE(0); + return 0; } // Return true if there are more items in the vector. - FUNC_LEAVE(iterator->index < pdc_vector_size(iterator->vector)); + return iterator->index < pdc_vector_size(iterator->vector); } \ No newline at end of file diff --git a/src/commons/file/common_io.c b/src/commons/file/common_io.c index 22831d539..f007ef308 100644 --- a/src/commons/file/common_io.c +++ b/src/commons/file/common_io.c @@ -4,10 +4,32 @@ #include #include #include + #include "pdc_logger.h" #include "pdc_timing.h" #include "pdc_malloc.h" +int +get_file_size(FILE *fp, size_t *file_size) +{ + + FUNC_ENTER(NULL); + + long current = ftell(fp); + errno = 0; + if ((*file_size = fseek(fp, 0, SEEK_END)) != 0) { + LOG_ERROR("%s\n", strerror(errno)); + FUNC_LEAVE(-1); + } + errno = 0; + if (fseek(fp, current, SEEK_SET) != 0) { + LOG_ERROR("%s\n", strerror(errno)); + FUNC_LEAVE(-1); + } + + FUNC_LEAVE(0); +} + FILE * open_file(char *filename, char *mode) { @@ -16,7 +38,7 @@ open_file(char *filename, char *mode) FILE *fp = fopen(filename, mode); if (fp == NULL) { LOG_ERROR("Error opening file %s: %s\n", filename, strerror(errno)); - exit(1); + FUNC_LEAVE(NULL); } FUNC_LEAVE(fp); diff --git a/src/commons/file/include/bin_file_ops.h b/src/commons/file/include/bin_file_ops.h index 3a6890234..4bb1dd08b 100644 --- a/src/commons/file/include/bin_file_ops.h +++ b/src/commons/file/include/bin_file_ops.h @@ -34,6 +34,4 @@ uint64_t *bin_read_uint64(FILE *file); size_t *bin_read_size_t(FILE *file); -size_t bin_skip_field(FILE *stream); - #endif /* !BIN_FILE_OPS_H */ diff --git a/src/commons/file/include/common_io.h b/src/commons/file/include/common_io.h index f6b248171..fac794037 100644 --- a/src/commons/file/include/common_io.h +++ b/src/commons/file/include/common_io.h @@ -88,6 +88,15 @@ void print_string(char *string); */ int read_line(FILE *fp, char *buffer, size_t size); +/** + * \fn int get_file_size(const char *filename, size_t file_size) + * \brief Get file size. File pointer is not changed. + * \param fp Pointer to the file to read from. + * \param file_size Pointer to be updated with filesize. + * \return 0 if the file_size was retrieved successfully, non-zero value otherwise. + */ +int get_file_size(FILE *fp, size_t *file_size); + /** * \fn int get_input(char *buffer, size_t size) * \brief Gets user input from the standard input into a buffer. diff --git a/src/commons/logging/include/pdc_logger.h b/src/commons/logging/include/pdc_logger.h index 161945e85..cb0151780 100644 --- a/src/commons/logging/include/pdc_logger.h +++ b/src/commons/logging/include/pdc_logger.h @@ -15,7 +15,7 @@ typedef enum { LOG_LEVEL_ERROR, LOG_LEVEL_WARNING, LOG_LEVEL_INFO, LOG_LEVEL_DEB static FILE * logFiles[4] = {NULL}; // Log files for each log level static char logFilenames[4][MAX_LOG_FILE_NAME_LENGTH]; -static PDC_LogLevel logLevel = LOG_LEVEL_INFO; +static PDC_LogLevel logLevel = LOG_LEVEL_WARNING; void setLogFile(PDC_LogLevel level, const char *fileName); void setLogLevel(PDC_LogLevel level); @@ -51,4 +51,4 @@ void log_message(bool is_server, bool just_print, PDC_LogLevel level, const char log_message(false, true, LOG_LEVEL_INFO, __FILE__, __func__, __LINE__, format, ##__VA_ARGS__) #endif -#endif // PDC_LOGGER_H \ No newline at end of file +#endif // PDC_LOGGER_H diff --git a/src/commons/logging/pdc_logger.c b/src/commons/logging/pdc_logger.c index ce3b3d05e..0c7b5c69d 100644 --- a/src/commons/logging/pdc_logger.c +++ b/src/commons/logging/pdc_logger.c @@ -132,13 +132,21 @@ _log_message(bool is_server, PDC_LogLevel level, const char *file, const char *f vsnprintf(message, MAX_LOG_MSG_LENGTH, format, args); #ifdef ENABLE_MPI - static int my_rank = -1; - if (my_rank == -1) - my_rank = PDC_get_rank(); + static int mpi_initialized = 0; + static int my_rank = -1; + + if (!mpi_initialized) + MPI_Initialized(&mpi_initialized); + + if (mpi_initialized) { + my_rank = -1; + if (my_rank == -1) + my_rank = PDC_get_rank(); + } #endif // Print differently based on log level - if (level == LOG_LEVEL_ERROR || level == LOG_LEVEL_DEBUG) { + if (level == LOG_LEVEL_ERROR || level == LOG_LEVEL_DEBUG || level == LOG_LEVEL_WARNING) { #ifdef ENABLE_MPI if (is_server) fprintf(logFile, "[%s.%06ld] [%s] [%s:%d] PDC_SERVER[%d]: %s", timestr, tv.tv_usec, prefix, diff --git a/src/commons/utils/include/pdc_mercury_auth.h b/src/commons/utils/include/pdc_mercury_auth.h new file mode 100644 index 000000000..ba79c2002 --- /dev/null +++ b/src/commons/utils/include/pdc_mercury_auth.h @@ -0,0 +1,23 @@ +#ifndef PDC_MERCURY_AUTH_H +#define PDC_MERCURY_AUTH_H + +#include + +#include "pdc_public.h" + +#define PDC_PERLMUTTER_CXI_ENV_COUNT 3 + +typedef struct pdc_scoped_env_entry_t { + const char *name; + char * old_value; + pbool_t had_old_value; +} pdc_scoped_env_entry_t; + +perr_t PDC_scope_perlmutter_cxi_auth_env(const char * hg_transport, + pdc_scoped_env_entry_t envs[PDC_PERLMUTTER_CXI_ENV_COUNT], + int *env_count, unsigned int *svc_id, unsigned int *vni, + char *device_buf, size_t device_buf_len); + +void PDC_restore_scoped_env(pdc_scoped_env_entry_t envs[PDC_PERLMUTTER_CXI_ENV_COUNT], int env_count); + +#endif diff --git a/src/commons/utils/pdc_malloc.c b/src/commons/utils/pdc_malloc.c index c9178328e..76883a7b7 100644 --- a/src/commons/utils/pdc_malloc.c +++ b/src/commons/utils/pdc_malloc.c @@ -60,6 +60,11 @@ PDC_malloc(size_t size) if (ret_value) PDC_mem_usage_g += size; + if (ret_value == NULL) { + LOG_ERROR("PDC_malloc failed to allocate %zu bytes\n", size); + abort(); + } + FUNC_LEAVE(ret_value); } @@ -93,6 +98,11 @@ PDC_calloc(size_t count, size_t size) if (ret_value) PDC_mem_usage_g += size; + if (ret_value == NULL) { + LOG_ERROR("PDC_calloc failed to allocate %zu bytes\n", count * size); + abort(); + } + FUNC_LEAVE(ret_value); } @@ -141,7 +151,14 @@ PDC_realloc(void *ptr, size_t size) #ifdef HAVE_MALLOC_USABLE_SIZE _old_size = malloc_usable_size(ptr); #endif - FUNC_LEAVE(PDC_realloc_knowing_oldsize(ptr, size, _old_size)); + + void *ret = PDC_realloc_knowing_oldsize(ptr, size, _old_size); + if (ret == NULL) { + LOG_ERROR("PDC_realloc failed to allocate %zu bytes\n", size); + abort(); + } + + FUNC_LEAVE(ret); } void * diff --git a/src/commons/utils/pdc_mercury_auth.c b/src/commons/utils/pdc_mercury_auth.c new file mode 100644 index 000000000..8628a3d8f --- /dev/null +++ b/src/commons/utils/pdc_mercury_auth.c @@ -0,0 +1,183 @@ +#include "pdc_mercury_auth.h" + +#include +#include +#include +#include + +#include "pdc_private.h" + +static pbool_t +PDC_is_cxi_mercury_transport(const char *hg_transport) +{ + return (hg_transport != NULL && + (strcmp(hg_transport, "ofi+cxi") == 0 || strcmp(hg_transport, "cxi") == 0)); +} + +static pbool_t +PDC_is_perlmutter_system(void) +{ + const char *nersc_host = getenv("NERSC_HOST"); + + return (nersc_host != NULL && strcmp(nersc_host, "perlmutter") == 0); +} + +static void +PDC_scope_env_set(pdc_scoped_env_entry_t *entry, const char *name, const char *value) +{ + const char *old_value = getenv(name); + + entry->name = name; + entry->had_old_value = (old_value != NULL) ? TRUE : FALSE; + entry->old_value = (old_value != NULL) ? strdup(old_value) : NULL; + + if (value != NULL) + setenv(name, value, 1); + else + unsetenv(name); +} + +static void +PDC_scope_env_restore(pdc_scoped_env_entry_t *entry) +{ + if (entry->had_old_value == TRUE) + setenv(entry->name, entry->old_value, 1); + else + unsetenv(entry->name); + + free(entry->old_value); + entry->old_value = NULL; +} + +static perr_t +PDC_discover_perlmutter_cxi_auth(unsigned int *svc_id, unsigned int *vni) +{ + perr_t ret_value = FAIL; + FILE * fp = NULL; + char line[256]; + char uid_pattern[64]; + + unsigned int current_svc_id = 0; + unsigned int current_vni = 0; + pbool_t enabled = FALSE; + pbool_t system_service = FALSE; + pbool_t member_match = FALSE; + + if (svc_id == NULL || vni == NULL) + PGOTO_ERROR(FAIL, "Invalid output pointers for CXI auth discovery\n"); + + snprintf(uid_pattern, sizeof(uid_pattern), "uid=%u", (unsigned int)geteuid()); + + fp = popen("cxi_service -v list 2>/dev/null", "r"); + if (fp == NULL) + PGOTO_ERROR(FAIL, "Could not execute cxi_service\n"); + + while (fgets(line, sizeof(line), fp) != NULL) { + unsigned int parsed_value = 0; + + if (sscanf(line, " ID: %u", &parsed_value) == 1) { + current_svc_id = parsed_value; + current_vni = 0; + enabled = FALSE; + system_service = FALSE; + member_match = FALSE; + continue; + } + + if (strstr(line, "Enabled") != NULL) { + enabled = (strstr(line, "Yes") != NULL) ? TRUE : FALSE; + continue; + } + + if (strstr(line, "System Service") != NULL) { + system_service = (strstr(line, "Yes") != NULL) ? TRUE : FALSE; + continue; + } + + if (strstr(line, "Valid Members") != NULL) { + member_match = (strstr(line, uid_pattern) != NULL) ? TRUE : FALSE; + continue; + } + + if (sscanf(line, " ---> Valid VNIs : %u", &parsed_value) == 1) { + current_vni = parsed_value; + if (enabled == TRUE && system_service == FALSE && member_match == TRUE) { + *svc_id = current_svc_id; + *vni = current_vni; + ret_value = SUCCEED; + break; + } + } + } + +done: + if (fp != NULL) + pclose(fp); + + return ret_value; +} + +perr_t +PDC_scope_perlmutter_cxi_auth_env(const char * hg_transport, + pdc_scoped_env_entry_t envs[PDC_PERLMUTTER_CXI_ENV_COUNT], int *env_count, + unsigned int *svc_id, unsigned int *vni, char *device_buf, + size_t device_buf_len) +{ + perr_t ret_value = SUCCEED; + const char * nic_name = getenv("HG_CXI_NIC"); + unsigned int local_svc_id; + unsigned int local_vni; + char svc_buf[32]; + char vni_buf[32]; + + if (env_count == NULL) + PGOTO_ERROR(FAIL, "Invalid env count pointer for CXI auth scoping\n"); + + *env_count = 0; + + if (svc_id != NULL) + *svc_id = 0; + if (vni != NULL) + *vni = 0; + if (device_buf != NULL && device_buf_len > 0) + device_buf[0] = '\0'; + + if (PDC_is_perlmutter_system() == FALSE || PDC_is_cxi_mercury_transport(hg_transport) == FALSE) + return SUCCEED; + + if (nic_name == NULL || nic_name[0] == '\0') + nic_name = "cxi0"; + + if (PDC_discover_perlmutter_cxi_auth(&local_svc_id, &local_vni) != SUCCEED) + PGOTO_ERROR(FAIL, "Unable to discover Perlmutter CXI service/VNI\n"); + + snprintf(svc_buf, sizeof(svc_buf), "%u", local_svc_id); + snprintf(vni_buf, sizeof(vni_buf), "%u", local_vni); + + PDC_scope_env_set(&envs[(*env_count)++], "SLINGSHOT_VNIS", vni_buf); + PDC_scope_env_set(&envs[(*env_count)++], "SLINGSHOT_SVC_IDS", svc_buf); + PDC_scope_env_set(&envs[(*env_count)++], "SLINGSHOT_DEVICES", nic_name); + + if (svc_id != NULL) + *svc_id = local_svc_id; + if (vni != NULL) + *vni = local_vni; + if (device_buf != NULL && device_buf_len > 0) { + snprintf(device_buf, device_buf_len, "%s", nic_name); + } + +done: + if (ret_value != SUCCEED) + PDC_restore_scoped_env(envs, *env_count); + + return ret_value; +} + +void +PDC_restore_scoped_env(pdc_scoped_env_entry_t envs[PDC_PERLMUTTER_CXI_ENV_COUNT], int env_count) +{ + while (env_count > 0) { + env_count--; + PDC_scope_env_restore(&envs[env_count]); + } +} diff --git a/src/server/CMakeLists.txt b/src/server/CMakeLists.txt index c1e7bb0d5..8cf3b9267 100644 --- a/src/server/CMakeLists.txt +++ b/src/server/CMakeLists.txt @@ -37,15 +37,15 @@ include_directories( ${PDC_SOURCE_DIR}/src/server/include ${PDC_SOURCE_DIR}/src/server/pdc_server_region ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include - ${PDC_SOURCE_DIR}/src/server/pdc_server_analysis/include ${PDC_SOURCE_DIR}/src/server/dablooms + ${PDC_SOURCE_DIR}/src/server/transform/include ${PDC_SOURCE_DIR}/src/api/include ${PDC_SOURCE_DIR}/src/api/pdc_obj/include ${PDC_SOURCE_DIR}/src/api/pdc_region/include + ${PDC_SOURCE_DIR}/src/api/pdc_tf/include ${PDC_SOURCE_DIR}/src/api/pdc_query/include - ${PDC_SOURCE_DIR}/src/api/pdc_transform/include - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/include ${PDC_SOURCE_DIR}/src/api/profiling/include + ${PDC_SOURCE_DIR}/src/api/pdc_hist/include ${PDC_SOURCE_DIR}/src/utils/include ${MERCURY_INCLUDE_DIR} ${FASTBIT_INCLUDE_DIR} @@ -65,18 +65,25 @@ add_library(pdc_server_lib pdc_server_metadata_index.c pdc_server_metadata.c pdc_client_server_common.c + transform/pdc_tf_user.c + transform/pdc_tf_server.c + transform/pdc_tf_common.c + transform/pdc_tf_poly_sched.c + transform/pdc_tf_builtin_zfp_cpu.c + transform/pdc_tf_builtin_zfp_gpu.c + transform/pdc_tf_builtin_sz_cpu.c + transform/pdc_tf_builtin_sz_gpu.c + transform/pdc_tf_builtin_encrypt_cpu.c + transform/pdc_tf_builtin_turbo_cpu.c + transform/pdc_tf_profiler.c dablooms/pdc_dablooms.c dablooms/pdc_murmur.c - # pdc_hash-table.c - ${PDC_SOURCE_DIR}/src/server/pdc_server_analysis/pdc_server_analysis.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_data.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_cache.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer.c ${PDC_SOURCE_DIR}/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c ${PDC_SOURCE_DIR}/src/utils/pdc_region_utils.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_analysis_common.c - ${PDC_SOURCE_DIR}/src/api/pdc_transform/pdc_transforms_common.c - ${PDC_SOURCE_DIR}/src/api/pdc_analysis/pdc_hist_pkg.c + ${PDC_SOURCE_DIR}/src/api/pdc_hist/pdc_hist_pkg.c ) if(PDC_ENABLE_FASTBIT) message(STATUS "Enabled fastbit") diff --git a/src/server/include/pdc_client_server_common.h b/src/server/include/pdc_client_server_common.h index 92da04254..5fa3725dd 100644 --- a/src/server/include/pdc_client_server_common.h +++ b/src/server/include/pdc_client_server_common.h @@ -29,9 +29,9 @@ #include "pdc_private.h" #include "pdc_linkedlist.h" #include "pdc_prop_pkg.h" -#include "pdc_analysis_and_transforms_common.h" #include "pdc_query.h" #include "pdc_malloc.h" +#include "pdc_tf_common.h" #include "pdc_timing.h" #include "pdc_server_region_transfer_metadata_query.h" @@ -217,15 +217,6 @@ typedef struct pdc_metadata_transfer_t { uint32_t data_server_id; uint8_t region_partition; uint8_t consistency; - - // The following support state changes to objects - // as a result of a transform. - int32_t current_state; - int8_t t_storage_order; - int8_t t_dtype; - size_t t_ndim; - uint64_t t_dims0, t_dims1, t_dims2, t_dims3; - int t_meta_index; } pdc_metadata_transfer_t; typedef struct { @@ -309,6 +300,10 @@ typedef struct pdc_data_server_io_list_t { struct pdc_data_server_io_list_t *next; } pdc_data_server_io_list_t; +/** + * This data structure is used for STORE_REGION_BY_REGION_SINGLE_FILE strategy + * It has the metadata needed for checkpoint and restart + */ typedef struct data_server_region_t { uint64_t obj_id; int fd; // file handle @@ -323,10 +318,7 @@ typedef struct data_server_region_t { // For region map region_map_t *region_map_head; // For region storage - region_list_t *region_storage_head; - // For non-mapped object analysis - // Used primarily as a local_temp - void * obj_data_ptr; + region_list_t * region_storage_head; char * storage_location; // save the file location to enable reopening struct data_server_region_t *prev; struct data_server_region_t *next; @@ -379,10 +371,6 @@ typedef struct pdc_metadata_t { // For buf to obj map region_buf_map_t *region_buf_map_head; - // For transforms - int transform_state; - struct _pdc_transform_state current_state; - pdc_histogram_t *obj_hist; // For hash table list @@ -549,57 +537,6 @@ typedef struct { uint8_t lock_mode; } region_lock_in_t; -/* FIXME: The bulk_args->in structure (shown below) appears is defined as a - * superset of the region_lock_in_t structure. That way, we can simply pass - * a pointer to our version into the PDC_Data_Server_release function... - */ -/* Define region_transform_and_lock_in_t */ -typedef struct { - uint32_t meta_server_id; - uint64_t obj_id; - uint8_t access_type; - pdcid_t local_reg_id; - region_info_transfer_t region; - uint8_t mapping; - uint8_t data_type; - uint8_t lock_mode; - /* The following are unique to the transform portion of the lock */ - uint8_t dest_type; - int32_t transform_id; - int32_t transform_state; - hg_bulk_t local_bulk_handle; - int64_t transform_data_size; - uint64_t client_data_ptr; - -} region_transform_and_lock_in_t; - -/* FIXME: The region_analysis structure (shown below) is defined as a - * superset of the region_lock_in_t structure. That way, we can simply pass - * a pointer to our version into the PDC_Data_Server_release function... - */ -/* Define region_analysis_and_lock_in_t */ -typedef struct { - uint32_t meta_server_id; - uint64_t obj_id; - uint8_t access_type; - pdcid_t local_reg_id; - region_info_transfer_t region; - uint8_t mapping; - uint8_t data_type; - uint8_t lock_mode; - /* ------------------------------------------------------------- - * The following are unique to the analysis portion of the lock - * ------------------------------------------------------------- - */ - int32_t analysis_meta_index; - int32_t n_args; - uint64_t output_obj_id; - uint64_t type_extent; - uint64_t output_type_extent; - pdcid_t input_iter; - pdcid_t output_iter; -} region_analysis_and_lock_in_t; - /* Define metadata_delete_in_t */ typedef struct { hg_string_t obj_name; @@ -787,8 +724,10 @@ typedef struct { size_t remote_unit; int32_t obj_ndim; uint32_t meta_server_id; + uint8_t access_type; - uint8_t access_type; + // For region transformations + pdc_tf_pkg_t pdc_tf_pkg; } transfer_request_in_t; /* Define transfer_request_out_t */ @@ -800,11 +739,10 @@ typedef struct { /* Define transfer_request_all_in_t */ typedef struct transfer_request_all_in_t { hg_bulk_t local_bulk_handle; - // hg_bulk_t local_bulk_handle2; - uint64_t total_buf_size; - int32_t n_objs; - uint8_t access_type; - int client_id; + uint64_t total_buf_size; + int32_t n_objs; + uint8_t access_type; + int client_id; } transfer_request_all_in_t; /* Define transfer_request_all_out_t */ @@ -816,10 +754,9 @@ typedef struct transfer_request_all_out_t { /* Define transfer_request_metadata_query_in_t */ typedef struct transfer_request_metadata_query_in_t { hg_bulk_t local_bulk_handle; - // hg_bulk_t local_bulk_handle2; - uint64_t total_buf_size; - int32_t n_objs; - uint8_t is_write; + uint64_t total_buf_size; + int32_t n_objs; + uint8_t is_write; } transfer_request_metadata_query_in_t; /* Define transfer_request_metadata_query_out_t */ @@ -1323,148 +1260,6 @@ hg_proc_region_lock_out_t(hg_proc_t proc, void *data) FUNC_LEAVE(ret); } -/* Define hg_proc_region_transform_and_lock_in_t */ -static HG_INLINE hg_return_t -hg_proc_region_transform_and_lock_in_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - region_transform_and_lock_in_t *struct_data = (region_transform_and_lock_in_t *)data; - - ret = hg_proc_int32_t(proc, &struct_data->meta_server_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->obj_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->access_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->local_reg_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_region_info_transfer_t(proc, &struct_data->region); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->mapping); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->data_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->lock_mode); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->dest_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->transform_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->transform_state); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_hg_bulk_t(proc, &struct_data->local_bulk_handle); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int64_t(proc, &struct_data->transform_data_size); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->client_data_ptr); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - -/* Define hg_proc_region_analysis_and_lock_in_t */ -static HG_INLINE hg_return_t -hg_proc_region_analysis_and_lock_in_t(hg_proc_t proc, void *data) -{ - FUNC_ENTER(NULL); - - hg_return_t ret; - region_analysis_and_lock_in_t *struct_data = (region_analysis_and_lock_in_t *)data; - - ret = hg_proc_int32_t(proc, &struct_data->meta_server_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->obj_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->access_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->local_reg_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_region_info_transfer_t(proc, &struct_data->region); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->mapping); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->data_type); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint8_t(proc, &struct_data->lock_mode); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->analysis_meta_index); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->n_args); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->output_obj_id); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->type_extent); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->output_type_extent); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->input_iter); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->output_iter); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - - FUNC_LEAVE(ret); -} - /* Define hg_proc_metadata_query_transfer_in_t */ static HG_INLINE hg_return_t hg_proc_metadata_query_transfer_in_t(hg_proc_t proc, void *data) @@ -1628,43 +1423,6 @@ hg_proc_pdc_metadata_transfer_t(hg_proc_t proc, void *data) if (ret != HG_SUCCESS) { FUNC_LEAVE(ret); } - // Added to support transforms - ret = hg_proc_int32_t(proc, &struct_data->current_state); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int8_t(proc, &struct_data->t_storage_order); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int8_t(proc, &struct_data->t_dtype); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_hg_size_t(proc, &struct_data->t_ndim); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->t_dims0); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->t_dims1); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->t_dims2); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_uint64_t(proc, &struct_data->t_dims3); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } - ret = hg_proc_int32_t(proc, &struct_data->t_meta_index); - if (ret != HG_SUCCESS) { - FUNC_LEAVE(ret); - } FUNC_LEAVE(ret); } @@ -2642,6 +2400,26 @@ hg_proc_transfer_request_in_t(hg_proc_t proc, void *data) if (ret != HG_SUCCESS) { FUNC_LEAVE(ret); } + ret = hg_proc_uint32_t(proc, &struct_data->pdc_tf_pkg.pdc_var_type); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_hg_string_t(proc, &struct_data->pdc_tf_pkg.json_filepath); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_hg_string_t(proc, &struct_data->pdc_tf_pkg.cur_state); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_hg_string_t(proc, &struct_data->pdc_tf_pkg.client_state); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } + ret = hg_proc_hg_string_t(proc, &struct_data->pdc_tf_pkg.store_state); + if (ret != HG_SUCCESS) { + FUNC_LEAVE(ret); + } FUNC_LEAVE(ret); } @@ -3988,52 +3766,6 @@ struct buf_map_release_bulk_args { region_lock_in_t in; }; -// The following two tructures are the same as: buf_map_release_bulk_args -// (above) with one modified field, i.e. the region_transform_and_lock_in_t -// and the region_analysis_and_lock_in_t rather than region_lock_in_t. -// This should allow the normal thread processing for IO operations -// to work correctly. N.B. I moved the region_lock_in_t structure in -// the struct buf_map_release_bulk_args above, to be the last element. -// Thus, the size difference in the last element shouldn't matter -// to normal processing of things... - -struct buf_map_transform_and_release_bulk_args { - hg_handle_t handle; - void * data_buf; - pdcid_t remote_obj_id; /* target of object id */ - pdcid_t remote_reg_id; /* target of region id */ - int32_t remote_client_id; - struct pdc_region_info * remote_reg_info; - region_info_transfer_t remote_region; - hg_bulk_t remote_bulk_handle; - hg_bulk_t local_bulk_handle; - hg_addr_t local_addr; - struct hg_thread_work work; - hg_thread_mutex_t work_mutex; - hg_thread_cond_t work_cond; - int work_completed; - region_transform_and_lock_in_t in; -}; - -struct buf_map_analysis_and_release_bulk_args { - hg_handle_t handle; - void * data_buf; - pdcid_t remote_obj_id; /* target of object id */ - pdcid_t remote_reg_id; /* target of region id */ - int32_t remote_client_id; - struct pdc_region_info *remote_reg_info; - region_info_transfer_t remote_region; - hg_bulk_t remote_bulk_handle; - hg_bulk_t local_bulk_handle; - hg_addr_t local_addr; - struct hg_thread_work work; - hg_thread_mutex_t work_mutex; - hg_thread_cond_t work_cond; - int work_completed; - /* region_analysis_and_lock_in_t is a superset of region_lock_in_t */ - region_analysis_and_lock_in_t in; -}; - struct lock_bulk_args { hg_handle_t handle; region_lock_in_t in; @@ -4221,9 +3953,6 @@ hg_id_t PDC_region_lock_register(hg_class_t *hg_class); hg_id_t PDC_data_server_write_register(hg_class_t *hg_class); hg_id_t PDC_notify_region_update_register(hg_class_t *hg_class); hg_id_t PDC_region_release_register(hg_class_t *hg_class); -hg_id_t PDC_region_analysis_release_register(hg_class_t *hg_class); -hg_id_t PDC_region_transform_release_register(hg_class_t *hg_class); -hg_id_t PDC_transform_region_release_register(hg_class_t *hg_class); hg_id_t PDC_buf_map_server_register(hg_class_t *hg_class); hg_id_t PDC_buf_unmap_server_register(hg_class_t *hg_class); diff --git a/src/server/pdc_client_server_common.c b/src/server/pdc_client_server_common.c index b75c8f567..601baa409 100644 --- a/src/server/pdc_client_server_common.c +++ b/src/server/pdc_client_server_common.c @@ -27,9 +27,6 @@ #include "pdc_client_server_common.h" #include "pdc_query.h" #include "pdc_region.h" -#include "pdc_transforms_pkg.h" -#include "pdc_analysis_pkg.h" -#include "pdc_analysis.h" #include "pdc_hist_pkg.h" #include "pdc_utlist.h" #include "pdc_server.h" @@ -260,6 +257,9 @@ PDC_get_var_type_size(pdc_var_type_t dtype) case PDC_UINT16: PGOTO_DONE(sizeof(uint16_t)); break; + case PDC_INT32: + PGOTO_DONE(sizeof(int32_t)); + break; case PDC_INT64: PGOTO_DONE(sizeof(int64_t)); break; @@ -826,15 +826,6 @@ PDC_metadata_t_to_transfer_t(pdc_metadata_t *meta, pdc_metadata_transfer_t *tran transfer->dims3 = meta->dims[3]; transfer->tags = meta->tags; transfer->data_location = meta->data_location; - transfer->current_state = meta->transform_state; - transfer->t_storage_order = meta->current_state.storage_order; - transfer->t_dtype = meta->current_state.dtype; - transfer->t_ndim = meta->current_state.ndim; - transfer->t_dims0 = meta->current_state.dims[0]; - transfer->t_dims1 = meta->current_state.dims[1]; - transfer->t_dims2 = meta->current_state.dims[2]; - transfer->t_dims3 = meta->current_state.dims[3]; - transfer->t_meta_index = meta->current_state.meta_index; done: FUNC_LEAVE(ret_value); @@ -869,20 +860,6 @@ PDC_transfer_t_to_metadata_t(pdc_metadata_transfer_t *transfer, pdc_metadata_t * strcpy(meta->tags, transfer->tags); strcpy(meta->data_location, transfer->data_location); - if ((meta->transform_state = transfer->current_state) == 0) { - memset(&meta->current_state, 0, sizeof(struct _pdc_transform_state)); - } - else { - meta->current_state.storage_order = transfer->t_storage_order; - meta->current_state.dtype = transfer->t_dtype; - meta->current_state.ndim = transfer->t_ndim; - meta->current_state.dims[0] = transfer->t_dims0; - meta->current_state.dims[1] = transfer->t_dims1; - meta->current_state.dims[2] = transfer->t_dims2; - meta->current_state.dims[3] = transfer->t_dims3; - meta->current_state.meta_index = transfer->t_meta_index; - } - done: FUNC_LEAVE(ret_value); } @@ -1648,7 +1625,8 @@ HG_TEST_RPC_CB(obj_reset_dims, handle) // Decode input HG_Get_input(handle, &in); - if (!try_reset_dims()) { + if (!can_reset_dims()) { + LOG_WARNING("Cannot resize object dims due to storage strategy\n"); out.ret = 0; PGOTO_DONE(ret_value); } @@ -2164,341 +2142,6 @@ pdc_region_read_from_progress(void *arg) } #endif -// enter this function, transfer is done, data is in data server -static hg_return_t -transform_and_region_release_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - region_lock_out_t out; - struct buf_map_transform_and_release_bulk_args *bulk_args = NULL; - void * data_buf; - char * buf; - int ndim; - int transform_id; - int type_extent = 0; - int use_transform_size = 0; - size_t unit = 1; - int64_t transform_size, expected_size; - uint64_t * dims = NULL; - pdc_var_type_t destType; - struct _pdc_region_transform_ftn_info ** registry = NULL; - int registered_count; -#ifdef ENABLE_MULTITHREAD - data_server_region_t *target_reg = NULL; - region_buf_map_t * elt; -#else - struct pdc_region_info *remote_reg_info = NULL; -#endif - - bulk_args = (struct buf_map_transform_and_release_bulk_args *)hg_cb_info->arg; - - if (hg_cb_info->ret == HG_CANCELED) { - out.ret = 0; - PGOTO_ERROR(HG_OTHER_ERROR, "HG_Bulk_transfer() was successfully canceled"); - } - else if (hg_cb_info->ret != HG_SUCCESS) { - out.ret = 0; - PGOTO_ERROR(HG_PROTOCOL_ERROR, "Error in region_release_bulk_transfer_cb()"); - } - buf = (char *)bulk_args->data_buf; - transform_size = bulk_args->in.transform_data_size; - - destType = bulk_args->in.dest_type; - if ((destType == PDC_INT) || (destType == PDC_UINT) || (destType == PDC_FLOAT)) - type_extent = sizeof(int); - else if ((destType == PDC_DOUBLE) || (destType == PDC_INT64) || (destType == PDC_UINT64)) - type_extent = sizeof(double); - else if (destType == PDC_INT16) - type_extent = sizeof(short); - else if (destType == PDC_INT8) - type_extent = 1; - - out.ret = 1; - HG_Respond(bulk_args->handle, NULL, NULL, &out); - - ndim = bulk_args->remote_region.ndim; - expected_size = PDC_get_region_desc_size_bytes(bulk_args->remote_region.count, type_extent, ndim); - - /* There are some transforms, e.g. type_casting in which the transform size - * will match the expected size. Other transforms such as compression - * will NOT match and as a result, we assume that the incoming - * data_buf is a one dimensional BTYE array. This eventually needs - * to be uncompressed into a region buffer having the remote_region shape... - * - * FIXME: It might be more convenient for everyone if the client provides - * the correct size and shape info in the case of compressed datasets. - * This raises an interesting question whether compressed datasets can be - * written correctly. - * - *** Example: if we assume that each client process which - * shares this server is writing data, but each with a different size, then - * how should the server manage this. One approach would be to assume - * that every rank has the same data extent, e.g. the non-compressed data - * length and fill those pre calculated regions on disk with the compressed - * data. But we know compression algorithms don't actually guarantee that - * data CAN be compressed and the result might actually be larger than the - * orginal un-transformed data! This would definitely cause a problem. - * - * The ONLY way that we can deal with such eventualities is to simply - * delay writing to disk until all clients provide their data. At that - * instant in time, the server "knows" the extent of each client buffer - * and could arrange to write them out in a single (large) operation... - */ - if (transform_size != expected_size) { - use_transform_size = 1; - unit = (size_t)type_extent; - } - if ((transform_id = bulk_args->in.transform_id) >= 0) { - /* load the transform - * and execute it using the received data as input. - * The output of the transform is used to fill the - * actual region data store... - */ - if ((data_buf = PDC_Server_get_region_buf_ptr(bulk_args->in.obj_id, bulk_args->in.region)) != NULL) { - registered_count = PDC_get_transforms(®istry); - if (use_transform_size) { - bulk_args->in.data_type = PDC_INT8; - dims = (uint64_t *)&transform_size; - ndim = 1; - } - - else { - /* Prepare for the transform */ - dims = (uint64_t *)PDC_calloc(ndim, sizeof(uint64_t)); - if (dims == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "TRANSFORM memory allocation failed"); - dims[0] = bulk_args->remote_region.count[0] / type_extent; - if (ndim > 1) - dims[1] = bulk_args->remote_region.count[1] / type_extent; - if (ndim > 2) - dims[2] = bulk_args->remote_region.count[2] / type_extent; - if (ndim > 3) - dims[3] = bulk_args->remote_region.count[3] / type_extent; - } - if ((registered_count >= transform_id) && (registry != NULL)) { - size_t (*this_transform)(void *, pdc_var_type_t, int, uint64_t *, void **, pdc_var_type_t) = - registry[transform_id]->ftnPtr; - size_t result = this_transform(buf, bulk_args->in.data_type, ndim, dims, &data_buf, - bulk_args->in.dest_type); - LOG_INFO("Transform returned %ld\n", result); - - if (use_transform_size == 0 && dims) - dims = (uint64_t *)PDC_free(dims); - use_transform_size = 0; - } - } - } - -#ifdef ENABLE_MULTITHREAD - bulk_args->work.func = pdc_region_write_out_progress; - bulk_args->work.args = bulk_args; - - target_reg = PDC_Server_get_obj_region(bulk_args->remote_obj_id); - DL_FOREACH(target_reg->region_buf_map_head, elt) - { - if ((bulk_args->remote_region).start[0] == elt->remote_region_unit.start[0] && - (bulk_args->remote_region).count[0] == elt->remote_region_unit.count[0]) { - // replace the count_0 value with the transform_size - if (use_transform_size) { - (bulk_args->remote_region).count[0] = transform_size; - (bulk_args->remote_region).ndim = 1; - } - elt->bulk_args = (struct buf_map_release_bulk_args *)bulk_args; - } - } - - hg_thread_pool_post(hg_test_thread_pool_fs_g, &(bulk_args->work)); -#else - remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - if (remote_reg_info == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "remote_reg_info memory allocation failed"); - - remote_reg_info->ndim = (bulk_args->remote_region).ndim; - remote_reg_info->offset = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - remote_reg_info->size = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - (remote_reg_info->offset)[0] = (bulk_args->remote_region).start[0]; - if (use_transform_size) - (remote_reg_info->size)[0] = transform_size; - else - (remote_reg_info->size)[0] = (bulk_args->remote_region).count[0]; - - PDC_Server_data_write_out(bulk_args->remote_obj_id, remote_reg_info, bulk_args->data_buf, unit); - PDC_Data_Server_region_release((region_lock_in_t *)&bulk_args->in, &out); - - PDC_Server_release_lock_request(bulk_args->remote_obj_id, remote_reg_info); -#endif - -done: -#ifndef ENABLE_MULTITHREAD - if (remote_reg_info) { - remote_reg_info->offset = (uint64_t *)PDC_free(remote_reg_info->offset); - remote_reg_info->size = (uint64_t *)PDC_free(remote_reg_info->size); - remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); - } - HG_Bulk_free(bulk_args->remote_bulk_handle); - HG_Free_input(bulk_args->handle, &(bulk_args->in)); - HG_Destroy(bulk_args->handle); - bulk_args = (struct buf_map_transform_and_release_bulk_args *)PDC_free(bulk_args); -#endif - - FUNC_LEAVE(ret_value); -} - -// enter this function, transfer is done, data is in data server -static hg_return_t -analysis_and_region_release_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - region_lock_out_t out; - struct buf_map_analysis_and_release_bulk_args *bulk_args = NULL; - void * data_buf; - int ndim; - uint64_t type_extent; - uint64_t * dims = NULL; - struct pdc_region_info * remote_reg_info = NULL; - struct pdc_region_info * local_reg_info = NULL; - double start_t, end_t, analysis_t, io_t; - double averages[4]; - - bulk_args = (struct buf_map_analysis_and_release_bulk_args *)hg_cb_info->arg; - - if (hg_cb_info->ret == HG_CANCELED) { - out.ret = 0; - PGOTO_ERROR(HG_OTHER_ERROR, "HG_Bulk_transfer() was successfully canceled"); - } - else if (hg_cb_info->ret != HG_SUCCESS) { - out.ret = 0; - PGOTO_ERROR(HG_PROTOCOL_ERROR, "Error in region_release_bulk_transfer_cb()"); - } - - /* Prepare for the transform */ - ndim = bulk_args->remote_region.ndim; - dims = (uint64_t *)PDC_calloc(ndim, sizeof(uint64_t)); - type_extent = bulk_args->in.type_extent; - /* FIXME: Support ONLY up to 4 dimensions */ - if (dims) { - if (ndim >= 1) - dims[0] = bulk_args->in.region.count[0] / type_extent; - if (ndim >= 2) - dims[1] = bulk_args->in.region.count[1] / type_extent; - if (ndim >= 3) - dims[2] = bulk_args->in.region.count[2] / type_extent; - if (ndim == 4) - dims[3] = bulk_args->in.region.count[3] / type_extent; - } - - out.ret = 1; - HG_Respond(bulk_args->handle, NULL, NULL, &out); - - /* load the Analysis function - * and execute it using the received data as input. - * The output of the transform is used to fill the - * actual region data store... - */ - - data_buf = PDC_Server_get_region_buf_ptr(bulk_args->remote_obj_id, bulk_args->remote_region); -#ifdef ENABLE_MPI - start_t = MPI_Wtime(); -#endif - if (data_buf != NULL) { - struct _pdc_region_analysis_ftn_info **registry = NULL; - struct _pdc_iterator_cbs_t iter_cbs = {PDCobj_data_getSliceCount, PDCobj_data_getNextBlock}; - int analysis_meta_index = bulk_args->in.analysis_meta_index; - - int registered_count = PDC_get_analysis_registry(®istry); - if ((registered_count >= analysis_meta_index) && (registry != NULL)) { - int (*analysis_ftn)(pdcid_t iterIn, pdcid_t iterOut, struct _pdc_iterator_cbs_t * _cbs) = - registry[analysis_meta_index]->ftnPtr; - int result = analysis_ftn(bulk_args->in.input_iter, bulk_args->in.output_iter, &iter_cbs); - LOG_INFO("Analysis returned %d\n", result); - } - } -#ifdef ENABLE_MPI - end_t = MPI_Wtime(); - analysis_t = end_t - start_t; - start_t = end_t; -#endif - remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - if (remote_reg_info == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "remote_reg_info memory allocation failed"); - - /* Here' we prepare to write the output data to disk... - * NOTE: I'll leave this "as is" for now, but the PDC_Server_data_write_out() - * function seems to only deal with 1D arrays. Internally, I'll convert - * the region size info into a 1D byte length to write the entire result. - * Note that size[0] retains the n_elements * type_extent; - * all other size arguments contain the number of elements without type_extent - * adjustments. In this way, the PDC_Server_data_write_out function can simply - * multiply all size argments to find the byte length of the data_buf... - */ - remote_reg_info->ndim = (bulk_args->remote_region).ndim; - remote_reg_info->offset = (uint64_t *)PDC_calloc(remote_reg_info->ndim, sizeof(uint64_t)); - remote_reg_info->size = (uint64_t *)PDC_calloc(remote_reg_info->ndim, sizeof(uint64_t)); - - PDC_copy_region_desc((bulk_args->remote_region).start, remote_reg_info->offset, remote_reg_info->ndim, - remote_reg_info->ndim); - PDC_copy_region_desc((bulk_args->remote_region).count, remote_reg_info->size, remote_reg_info->ndim, - remote_reg_info->ndim); - - /* Write the analysis results... */ - PDC_Server_data_write_out(bulk_args->remote_obj_id, remote_reg_info, data_buf, (size_t)type_extent); -#ifdef ENABLE_MPI - end_t = MPI_Wtime(); - io_t = end_t - start_t; -#endif - PDC_Data_Server_region_release((region_lock_in_t *)&bulk_args->in, &out); - local_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - if (local_reg_info == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "local_reg_info memory allocation failed"); - - local_reg_info->ndim = bulk_args->in.region.ndim; - local_reg_info->offset = (uint64_t *)PDC_calloc(local_reg_info->ndim, sizeof(uint64_t)); - local_reg_info->size = (uint64_t *)PDC_calloc(local_reg_info->ndim, sizeof(uint64_t)); - - PDC_copy_region_desc(bulk_args->in.region.start, local_reg_info->offset, local_reg_info->ndim, - local_reg_info->ndim); - PDC_copy_region_desc(bulk_args->in.region.count, local_reg_info->size, local_reg_info->ndim, - local_reg_info->ndim); - - PDC_Server_release_lock_request(bulk_args->in.obj_id, local_reg_info); - - averages[0] = analysis_t; - averages[1] = io_t; -#ifdef ENABLE_MPI - if (MPI_Reduce(&averages[0], &averages[2], 2, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD) == 0) { - int mpi_rank, mpi_size; - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - if (mpi_rank == 0) { - LOG_INFO("Analysis avg time = %lf seconds\nIO avg time = %lf\n", averages[2] / mpi_size, - averages[3] / mpi_size); - } - } -#endif - -done: -#ifndef ENABLE_MULTITHREAD - remote_reg_info->offset = (uint64_t *)PDC_free(remote_reg_info->offset); - remote_reg_info->size = (uint64_t *)PDC_free(remote_reg_info->size); - remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); - - local_reg_info->offset = (uint64_t *)PDC_free(local_reg_info->offset); - local_reg_info->size = (uint64_t *)PDC_free(local_reg_info->size); - local_reg_info = (struct pdc_region_info *)PDC_free(local_reg_info); - - HG_Bulk_free(bulk_args->remote_bulk_handle); - HG_Free_input(bulk_args->handle, &(bulk_args->in)); - HG_Destroy(bulk_args->handle); - bulk_args = (struct buf_map_analysis_and_release_bulk_args *)PDC_free(bulk_args); -#endif - - FUNC_LEAVE(ret_value); -} - // enter this function, transfer is done, data is in data server static hg_return_t buf_map_region_release_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) @@ -3021,840 +2664,6 @@ HG_TEST_RPC_CB(region_release, handle) FUNC_LEAVE(ret_value); } -static void -get_region_lock_in(region_transform_and_lock_in_t *in, region_lock_in_t *reg_lock_in) -{ - FUNC_ENTER(NULL); - - reg_lock_in->meta_server_id = in->meta_server_id; - reg_lock_in->obj_id = in->obj_id; - reg_lock_in->access_type = in->access_type; - reg_lock_in->local_reg_id = in->local_reg_id; - reg_lock_in->region = in->region; - reg_lock_in->mapping = in->mapping; - reg_lock_in->data_type = in->data_type; - reg_lock_in->lock_mode = in->lock_mode; - - FUNC_LEAVE_VOID(); -} - -static hg_return_t -region_read_transform_release(region_transform_and_lock_in_t *in, hg_handle_t handle) -{ - FUNC_ENTER(NULL); - - hg_return_t hg_ret, ret_value = HG_SUCCESS; - const struct hg_info *hg_info = NULL; - hg_bulk_t remote_bulk_handle = HG_BULK_NULL; - hg_uint32_t remote_count; - void * data_buf = NULL; - ; - void ** data_ptrs_to = NULL; - size_t * data_size_to = NULL; - region_buf_map_t *eltt2; - region_lock_out_t out; - - size_t size; - size_t unit; - int dirty_reg = 0; - - struct buf_map_transform_and_release_bulk_args *transform_release_bulk_args = NULL; - struct pdc_region_info * remote_reg_info = NULL; - data_server_region_t * target_obj = NULL; - region_list_t *tmp, *elt, *request_region = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - - hg_info = HG_Get_info(handle); - unit = PDC_get_var_type_size(in->data_type); - - PDC_region_transfer_t_to_list_t(&in->region, request_region); - target_obj = PDC_Server_get_obj_region(in->obj_id); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&lock_list_mutex_g); -#endif - DL_FOREACH(target_obj->region_lock_head, elt) - { - // check region is dirty or not, if dirty then transfer data - if ((PDC_is_same_region_list(request_region, elt)) == 1 && (elt->reg_dirty_from_buf == 1) && - (hg_atomic_get32(&(elt->buf_map_refcount)) == 0)) { - PGOTO_ERROR(HG_OTHER_ERROR, "Release %" PRId64 " access_type==READ (dirty) NOT SUPPORTED YET", - in->obj_id); - } - if ((PDC_is_same_region_list(request_region, elt) == 1) && (elt->reg_dirty_from_buf == 1) && - (hg_atomic_get32(&(elt->buf_map_refcount)) > 0)) { - dirty_reg = 1; - tmp = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - DL_FOREACH(target_obj->region_buf_map_head, eltt2) - { - PDC_region_transfer_t_to_list_t(&(eltt2->remote_region_unit), tmp); - if (PDC_is_same_region_list(tmp, request_region) == 1) { - transform_release_bulk_args = NULL; - remote_count = 1; - // get remote object memory addr - data_buf = PDC_Server_get_region_buf_ptr(in->obj_id, in->region); - data_ptrs_to = (void **)PDC_malloc(sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(sizeof(size_t)); - if ((data_ptrs_to == NULL) || (data_size_to == NULL)) { - PGOTO_ERROR(HG_OTHER_ERROR, "Memory allocation failed"); - } - data_ptrs_to[0] = data_buf; - - // See whether the transfer size is that of the region - // or whether we use the transform size. - if (in->transform_state && (in->transform_data_size > 0)) { - data_size_to[0] = in->transform_data_size; - unit = 1; - } - else - data_size_to[0] = (eltt2->remote_region_unit).count[0]; - - hg_ret = - HG_Bulk_create(hg_info->hg_class, remote_count, data_ptrs_to, - (hg_size_t *)data_size_to, HG_BULK_READWRITE, &remote_bulk_handle); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(hg_ret, "Could not create bulk data handle"); - - data_ptrs_to = (void **)PDC_free(data_ptrs_to); - data_size_to = (size_t *)PDC_free(data_size_to); - - remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - transform_release_bulk_args = - (struct buf_map_transform_and_release_bulk_args *)PDC_malloc( - sizeof(struct buf_map_transform_and_release_bulk_args)); - if ((remote_reg_info == NULL) || (transform_release_bulk_args == NULL)) - PGOTO_ERROR(HG_OTHER_ERROR, "Memory allocation failed"); - - transform_release_bulk_args->handle = handle; - transform_release_bulk_args->data_buf = data_buf; - transform_release_bulk_args->remote_obj_id = eltt2->remote_obj_id; - transform_release_bulk_args->remote_reg_id = eltt2->remote_reg_id; - transform_release_bulk_args->remote_reg_info = remote_reg_info; - transform_release_bulk_args->remote_region = eltt2->remote_region_unit; - transform_release_bulk_args->remote_client_id = eltt2->remote_client_id; - transform_release_bulk_args->remote_bulk_handle = remote_bulk_handle; - transform_release_bulk_args->local_bulk_handle = eltt2->local_bulk_handle; - transform_release_bulk_args->local_addr = eltt2->local_addr; - transform_release_bulk_args->in = *in; - - remote_reg_info->offset = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - remote_reg_info->size = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - if ((remote_reg_info->offset == NULL) || (remote_reg_info->size == NULL)) - PGOTO_ERROR(HG_OTHER_ERROR, "Memory allocation failed"); - - if (in->transform_state && (in->transform_data_size > 0)) { - remote_reg_info->ndim = 1; - (remote_reg_info->offset)[0] = (transform_release_bulk_args->remote_region).start[0]; - (remote_reg_info->size)[0] = in->transform_data_size; - transform_release_bulk_args->remote_region.count[0] = in->transform_data_size; - } - else { - remote_reg_info->ndim = (transform_release_bulk_args->remote_region).ndim; - (remote_reg_info->offset)[0] = (transform_release_bulk_args->remote_region).start[0]; - (remote_reg_info->size)[0] = (transform_release_bulk_args->remote_region).count[0]; - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_init(&(transform_release_bulk_args->work_mutex)); - hg_thread_cond_init(&(transform_release_bulk_args->work_cond)); - transform_release_bulk_args->work.func = pdc_region_read_from_progress; - transform_release_bulk_args->work.args = transform_release_bulk_args; - - eltt2->bulk_args = (struct buf_map_release_bulk_args *)transform_release_bulk_args; - - hg_thread_pool_post(hg_test_thread_pool_fs_g, &(transform_release_bulk_args->work)); - out.ret = 1; - HG_Respond(handle, NULL, NULL, &out); -#else - PDC_Server_data_read_from(transform_release_bulk_args->remote_obj_id, remote_reg_info, - data_buf, unit); - if (in->transform_state && (in->transform_data_size > 0)) - size = in->transform_data_size; - else - size = HG_Bulk_get_size(eltt2->local_bulk_handle); - if (size != HG_Bulk_get_size(remote_bulk_handle)) - PGOTO_ERROR(HG_OTHER_ERROR, "Local and remote bulk size does not match"); - - hg_ret = HG_Bulk_transfer(hg_info->context, obj_map_region_release_bulk_transfer_cb, - transform_release_bulk_args, HG_BULK_PUSH, hg_info->addr, - in->local_bulk_handle, 0, remote_bulk_handle, 0, size, - HG_OP_ID_IGNORE); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(hg_ret, "obj map could not write bulk data"); -#endif - } - } - tmp = (region_list_t *)PDC_free(tmp); - } - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&lock_list_mutex_g); -#endif - -done: - request_region = (region_list_t *)PDC_free(request_region); - if (dirty_reg == 0) { - // Perform lock release function - PDC_Data_Server_region_release((region_lock_in_t *)in, &out); - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - - FUNC_LEAVE(ret_value); -} - -/* transform_region_release_cb */ -HG_TEST_RPC_CB(transform_region_release, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - hg_return_t hg_ret; - region_transform_and_lock_in_t in; - region_lock_in_t reg_lock_in; - region_lock_out_t out; - const struct hg_info * hg_info = NULL; - data_server_region_t * target_obj; - int error = 0; - int dirty_reg = 0; - hg_size_t size; - void * data_buf; - region_list_t * elt, *request_region, *tmp; - struct buf_map_transform_and_release_bulk_args *buf_map_bulk_args = NULL; - hg_bulk_t remote_bulk_handle = HG_BULK_NULL; - region_buf_map_t * eltt; - hg_uint32_t remote_count; - void ** data_ptrs_to = NULL; - size_t * data_size_to = NULL; - - // Decode input - HG_Get_input(handle, &in); - /* Get info from handle */ - hg_info = HG_Get_info(handle); - - get_region_lock_in(&in, ®_lock_in); - if (in.access_type == PDC_READ) { - ret_value = region_read_transform_release(&in, handle); - } - // write lock release with mapping case - // do data transfer if it is write lock release with mapping. - else { - request_region = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - PDC_region_transfer_t_to_list_t(&in.region, request_region); - target_obj = PDC_Server_get_obj_region(in.obj_id); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&lock_list_mutex_g); -#endif - DL_FOREACH(target_obj->region_lock_head, elt) - { - if (PDC_is_same_region_list(request_region, elt) == 1 && elt->reg_dirty_from_buf == 1 && - hg_atomic_get32(&(elt->buf_map_refcount)) > 0) { - dirty_reg = 1; - tmp = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - DL_FOREACH(target_obj->region_buf_map_head, eltt) - { - PDC_region_transfer_t_to_list_t(&(eltt->remote_region_unit), tmp); - if (PDC_is_same_region_list(tmp, request_region) == 1) { - data_buf = PDC_malloc(in.transform_data_size); - remote_count = 1; - data_ptrs_to = (void **)PDC_malloc(sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(sizeof(size_t)); - if ((data_buf == NULL) || (data_ptrs_to == NULL) || (data_size_to == NULL)) - PGOTO_ERROR(HG_OTHER_ERROR, "Memory allocation failed"); - - *data_ptrs_to = data_buf; - *data_size_to = in.transform_data_size; - - /* Create a new block handle to read the data */ - hg_ret = - HG_Bulk_create(hg_info->hg_class, remote_count, data_ptrs_to, - (hg_size_t *)data_size_to, HG_BULK_READWRITE, &remote_bulk_handle); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "Could not create bulk data handle"); - } - /* Args that get passed to the callback function */ - buf_map_bulk_args = (struct buf_map_transform_and_release_bulk_args *)PDC_malloc( - sizeof(struct buf_map_transform_and_release_bulk_args)); - if (buf_map_bulk_args == NULL) { - PGOTO_ERROR(HG_OTHER_ERROR, "buf_map_bulk_args memory allocation failed"); - } - memset(buf_map_bulk_args, 0, sizeof(struct buf_map_transform_and_release_bulk_args)); - buf_map_bulk_args->handle = handle; - buf_map_bulk_args->data_buf = data_buf; - buf_map_bulk_args->in = in; - buf_map_bulk_args->remote_obj_id = eltt->remote_obj_id; - buf_map_bulk_args->remote_reg_id = eltt->remote_reg_id; - buf_map_bulk_args->remote_region = eltt->remote_region_unit; - buf_map_bulk_args->remote_client_id = eltt->remote_client_id; - buf_map_bulk_args->remote_bulk_handle = remote_bulk_handle; -#ifdef ENABLE_MULTITHREAD - - hg_thread_mutex_init(&(buf_map_bulk_args->work_mutex)); - hg_thread_cond_init(&(buf_map_bulk_args->work_cond)); -#endif - data_ptrs_to = (void **)PDC_free(data_ptrs_to); - data_size_to = (size_t *)PDC_free(data_size_to); - - /* Pull bulk data */ - size = in.transform_data_size; - hg_ret = - HG_Bulk_transfer(hg_info->context, /* Context */ - transform_and_region_release_bulk_transfer_cb, /* Callback */ - buf_map_bulk_args, /* args */ - HG_BULK_PULL, /* OP */ - hg_info->addr, /* Origin addr */ - in.local_bulk_handle, 0, /* Origin handle and offset */ - - remote_bulk_handle, 0, /* Local handle and offset */ - size, HG_OP_ID_IGNORE); /* */ - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(HG_OTHER_ERROR, "buf map could not read bulk data"); - } - } - } - tmp = (region_list_t *)PDC_free(tmp); - } - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&lock_list_mutex_g); -#endif - - request_region = (region_list_t *)PDC_free(request_region); - - if (dirty_reg == 0) { - // Perform lock release function - PDC_Data_Server_region_release(®_lock_in, &out); - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - } - -done: - if (error == 1) { - out.ret = 0; - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - - FUNC_LEAVE(ret_value); -} - -/* region_transform_release_cb */ -HG_TEST_RPC_CB(region_transform_release, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - hg_return_t hg_ret; - region_lock_in_t in; - region_lock_out_t out; - const struct hg_info *hg_info = NULL; - data_server_region_t *target_obj; - int error = 0; - int dirty_reg = 0; - hg_size_t size, remote_size; - ; - void * data_buf; - region_list_t * elt, *request_region, *tmp; - struct buf_map_release_bulk_args *buf_map_bulk_args = NULL; - hg_bulk_t remote_bulk_handle = HG_BULK_NULL; - region_buf_map_t * eltt; - hg_uint32_t k, remote_count = 0; - void ** data_ptrs_to = NULL; - size_t * data_size_to = NULL; - size_t type_size = 0; - ; - size_t dims[4] = {0, 0, 0, 0}; - - // Decode input - HG_Get_input(handle, &in); - /* Get info from handle */ - - hg_info = HG_Get_info(handle); - - if (in.access_type == PDC_READ) - PGOTO_ERROR(HG_OTHER_ERROR, "release %" PRId64 " access_type==READ NOT SUPPORTED YET", in.obj_id); - - // ************************************************************ - // write lock release with mapping case - // do data transfer if it is write lock release with mapping. - // ************************************************************ - else { - LOG_INFO("region_release_cb: release obj_id=%" PRIu64 " access_type==WRITE\n", in.obj_id); - request_region = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - PDC_region_transfer_t_to_list_t(&in.region, request_region); - target_obj = PDC_Server_get_obj_region(in.obj_id); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&lock_list_mutex_g); -#endif - DL_FOREACH(target_obj->region_lock_head, elt) - { - if (PDC_is_same_region_list(request_region, elt) == 1 && elt->reg_dirty_from_buf == 1 && - hg_atomic_get32(&(elt->buf_map_refcount)) > 0) { - dirty_reg = 1; - tmp = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - DL_FOREACH(target_obj->region_buf_map_head, eltt) - { - PDC_region_transfer_t_to_list_t(&(eltt->remote_region_unit), tmp); - if (PDC_is_same_region_list(tmp, request_region) == 1) { - // get remote object memory addr - type_size = eltt->remote_unit; - data_buf = PDC_Server_get_region_buf_ptr(in.obj_id, in.region); - if (in.region.ndim == 1) { - dims[0] = (eltt->remote_region_unit).count[0] / type_size; - remote_count = 1; - data_ptrs_to = (void **)PDC_malloc(sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(sizeof(size_t)); - *data_ptrs_to = data_buf; - *data_size_to = (eltt->remote_region_unit).count[0]; - } - - else if (in.region.ndim == 2) { - dims[1] = (eltt->remote_region_unit).count[1] / type_size; - remote_count = (eltt->remote_region_nounit).count[0]; - data_ptrs_to = (void **)PDC_malloc(remote_count * sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(remote_count * sizeof(size_t)); - data_ptrs_to[0] = data_buf + - type_size * dims[1] * (eltt->remote_region_nounit).start[0] + - (eltt->remote_region_nounit).start[0]; - data_size_to[0] = (eltt->remote_region_unit).count[0]; - for (k = 1; k < remote_count; k++) { - data_ptrs_to[k] = data_ptrs_to[k - 1] + eltt->remote_unit * dims[1]; - data_size_to[k] = data_size_to[0]; - } - } - - /* Create a new block handle to read the data */ - hg_ret = - HG_Bulk_create(hg_info->hg_class, remote_count, data_ptrs_to, - (hg_size_t *)data_size_to, HG_BULK_READWRITE, &remote_bulk_handle); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(FAIL, "Could not create bulk data handle"); - } - data_ptrs_to = (void **)PDC_free(data_ptrs_to); - data_size_to = (size_t *)PDC_free(data_size_to); - - buf_map_bulk_args = (struct buf_map_release_bulk_args *)PDC_malloc( - sizeof(struct buf_map_release_bulk_args)); - if (buf_map_bulk_args == NULL) { - PGOTO_ERROR(HG_OTHER_ERROR, "buf_map_bulk_args memory allocation failed"); - } - memset(buf_map_bulk_args, 0, sizeof(struct buf_map_release_bulk_args)); - buf_map_bulk_args->handle = handle; - buf_map_bulk_args->data_buf = data_buf; - buf_map_bulk_args->in = in; - buf_map_bulk_args->remote_obj_id = eltt->remote_obj_id; - buf_map_bulk_args->remote_reg_id = eltt->remote_reg_id; - buf_map_bulk_args->remote_region_unit = eltt->remote_region_unit; - buf_map_bulk_args->remote_region_nounit = eltt->remote_region_nounit; - buf_map_bulk_args->remote_client_id = eltt->remote_client_id; - buf_map_bulk_args->remote_bulk_handle = remote_bulk_handle; -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_init(&(buf_map_bulk_args->work_mutex)); - hg_thread_cond_init(&(buf_map_bulk_args->work_cond)); -#endif - /* Pull bulk data */ - size = HG_Bulk_get_size(eltt->local_bulk_handle); - remote_size = HG_Bulk_get_size(remote_bulk_handle); - if (size != remote_size) { - size = remote_size; - } - hg_ret = HG_Bulk_transfer(hg_info->context, buf_map_region_release_bulk_transfer_cb, - buf_map_bulk_args, HG_BULK_PULL, eltt->local_addr, - eltt->local_bulk_handle, 0, remote_bulk_handle, 0, size, - HG_OP_ID_IGNORE); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "buf map could not read bulk data"); - } - } - } - tmp = (region_list_t *)PDC_free(tmp); - } - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&lock_list_mutex_g); -#endif - request_region = (region_list_t *)PDC_free(request_region); - - if (dirty_reg == 0) { - // Perform lock release function - PDC_Data_Server_region_release(&in, &out); - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - } - -done: - if (error == 1) { - out.ret = 0; - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - - FUNC_LEAVE(ret_value); -} - -/* region_analysis_release_cb */ -HG_TEST_RPC_CB(region_analysis_release, handle) -{ - FUNC_ENTER(NULL); - - hg_return_t ret_value = HG_SUCCESS; - union { - region_analysis_and_lock_in_t analysis; - region_lock_in_t lock_release; - } in; - - region_lock_out_t out; - const struct hg_info * hg_info = NULL; - data_server_region_t * target_obj; - data_server_region_t * lock_obj; - int error = 0; - int dirty_reg = 0; - hg_size_t size; - hg_return_t hg_ret; - void * data_buf; - struct pdc_region_info * server_region; - region_list_t * elt, *request_region, *tmp; - struct region_lock_update_bulk_args * lock_update_bulk_args = NULL; - struct buf_map_analysis_and_release_bulk_args *buf_map_bulk_args = NULL, *obj_map_bulk_args = NULL; - hg_bulk_t lock_local_bulk_handle = HG_BULK_NULL; - hg_bulk_t remote_bulk_handle = HG_BULK_NULL; - struct pdc_region_info * remote_reg_info; - region_buf_map_t * eltt, *eltt2; - - hg_uint32_t k, remote_count = 0; - void ** data_ptrs_to = NULL; - size_t * data_size_to = NULL; - size_t type_size = 0; - size_t dims[4] = {0, 0, 0, 0}; - - // Decode input - HG_Get_input(handle, &in.lock_release); - /* Get info from handle */ - hg_info = HG_Get_info(handle); - - if (in.lock_release.access_type == PDC_READ) { - // check region is dirty or not, if dirty transfer data - request_region = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - PDC_region_transfer_t_to_list_t(&in.lock_release.region, request_region); - target_obj = PDC_Server_get_obj_region(in.lock_release.obj_id); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&lock_list_mutex_g); -#endif - DL_FOREACH(target_obj->region_lock_head, elt) - { - if (PDC_is_same_region_list(request_region, elt) == 1 && elt->reg_dirty_from_buf == 1 && - hg_atomic_get32(&(elt->buf_map_refcount)) == 0) { - dirty_reg = 1; - size = HG_Bulk_get_size(elt->bulk_handle); - data_buf = (void *)PDC_calloc(1, size); - // data transfer - server_region = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - server_region->ndim = 1; - server_region->size = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - server_region->offset = (uint64_t *)PDC_malloc(sizeof(uint64_t)); - (server_region->size)[0] = size; - (server_region->offset)[0] = in.lock_release.region.start[0]; - ret_value = PDC_Server_data_read_direct(elt->from_obj_id, server_region, data_buf); - if (ret_value != SUCCEED) - PGOTO_ERROR(HG_OTHER_ERROR, "Error with PDC_Server_data_read_direct"); - hg_ret = HG_Bulk_create(hg_info->hg_class, 1, &data_buf, &size, HG_BULK_READWRITE, - &lock_local_bulk_handle); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(hg_ret, "Could not create bulk data handle"); - - lock_update_bulk_args = (struct region_lock_update_bulk_args *)PDC_malloc( - sizeof(struct region_lock_update_bulk_args)); - lock_update_bulk_args->handle = handle; - lock_update_bulk_args->in = in.lock_release; - lock_update_bulk_args->remote_obj_id = elt->obj_id; - lock_update_bulk_args->remote_reg_id = elt->reg_id; - lock_update_bulk_args->remote_client_id = elt->client_id; - lock_update_bulk_args->data_buf = data_buf; - lock_update_bulk_args->server_region = server_region; - - hg_ret = HG_Bulk_transfer(hg_info->context, region_release_update_bulk_transfer_cb, - lock_update_bulk_args, HG_BULK_PUSH, elt->addr, elt->bulk_handle, 0, - lock_local_bulk_handle, 0, size, HG_OP_ID_IGNORE); - if (hg_ret != HG_SUCCESS) - PGOTO_ERROR(hg_ret, "Could not write bulk data"); - } - - if (PDC_is_same_region_list(request_region, elt) == 1 && elt->reg_dirty_from_buf == 1 && - hg_atomic_get32(&(elt->buf_map_refcount)) > 0) { - dirty_reg = 1; - tmp = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - DL_FOREACH(target_obj->region_buf_map_head, eltt2) - { - PDC_region_transfer_t_to_list_t(&(eltt2->remote_region_unit), tmp); - if (PDC_is_same_region_shape(tmp, in.analysis.output_type_extent, request_region, - - in.analysis.type_extent) == 1) { - // get remote object memory addr - data_buf = - PDC_Server_get_region_buf_ptr(in.lock_release.obj_id, in.lock_release.region); - if (in.lock_release.region.ndim == 1) { - remote_count = 1; - data_ptrs_to = (void **)PDC_malloc(sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(sizeof(size_t)); - *data_ptrs_to = data_buf; - *data_size_to = (eltt2->remote_region_unit).count[0]; - } - - hg_ret = - HG_Bulk_create(hg_info->hg_class, remote_count, data_ptrs_to, - (hg_size_t *)data_size_to, HG_BULK_READWRITE, &remote_bulk_handle); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "obj map could not create bulk data handle"); - } - data_ptrs_to = (void **)PDC_free(data_ptrs_to); - data_size_to = (size_t *)PDC_free(data_size_to); - - remote_reg_info = - (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - if (remote_reg_info == NULL) { - error = 1; - PGOTO_ERROR(HG_OTHER_ERROR, "remote_reg_info memory allocation failed"); - } - - obj_map_bulk_args = (struct buf_map_analysis_and_release_bulk_args *)PDC_malloc( - sizeof(struct buf_map_analysis_and_release_bulk_args)); - // memset(obj_map_bulk_args, 0, sizeof(struct buf_map_release_bulk_args)); - memset(obj_map_bulk_args, 0, sizeof(struct buf_map_analysis_and_release_bulk_args)); - obj_map_bulk_args->handle = handle; - obj_map_bulk_args->data_buf = data_buf; - obj_map_bulk_args->in = in.analysis; - obj_map_bulk_args->remote_obj_id = eltt2->remote_obj_id; - obj_map_bulk_args->remote_reg_id = eltt2->remote_reg_id; - obj_map_bulk_args->remote_reg_info = remote_reg_info; - obj_map_bulk_args->remote_region = eltt2->remote_region_unit; - obj_map_bulk_args->remote_client_id = eltt2->remote_client_id; - obj_map_bulk_args->remote_bulk_handle = remote_bulk_handle; - obj_map_bulk_args->local_bulk_handle = eltt2->local_bulk_handle; - obj_map_bulk_args->local_addr = eltt2->local_addr; - - remote_reg_info->ndim = (obj_map_bulk_args->remote_region).ndim; - remote_reg_info->offset = - (uint64_t *)PDC_malloc(remote_reg_info->ndim * sizeof(uint64_t)); - remote_reg_info->size = - (uint64_t *)PDC_malloc(remote_reg_info->ndim * sizeof(uint64_t)); - - PDC_copy_region_desc((obj_map_bulk_args->remote_region).start, - remote_reg_info->offset, remote_reg_info->ndim, - remote_reg_info->ndim); - PDC_copy_region_desc((obj_map_bulk_args->remote_region).count, remote_reg_info->size, - remote_reg_info->ndim, remote_reg_info->ndim); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_init(&(obj_map_bulk_args->work_mutex)); - hg_thread_cond_init(&(obj_map_bulk_args->work_cond)); - obj_map_bulk_args->work.func = pdc_region_read_from_progress; - obj_map_bulk_args->work.args = obj_map_bulk_args; - - eltt2->bulk_args = (struct buf_map_release_bulk_args *)obj_map_bulk_args; - - hg_thread_pool_post(hg_test_thread_pool_fs_g, &(obj_map_bulk_args->work)); - - out.ret = 1; - - HG_Respond(handle, NULL, NULL, &out); -#else - PDC_Server_data_read_from(obj_map_bulk_args->remote_obj_id, remote_reg_info, data_buf, - in.analysis.type_extent); - - size = HG_Bulk_get_size(eltt2->local_bulk_handle); - if (size != HG_Bulk_get_size(remote_bulk_handle)) { - error = 1; - PGOTO_ERROR(HG_OTHER_ERROR, "remote bulk size does not match\n"); - } - - hg_ret = HG_Bulk_transfer(hg_info->context, obj_map_region_release_bulk_transfer_cb, - obj_map_bulk_args, HG_BULK_PUSH, eltt2->local_addr, - eltt2->local_bulk_handle, 0, remote_bulk_handle, 0, size, - HG_OP_ID_IGNORE); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "obj map could not write bulk data"); - } -#endif - } - } - tmp = (region_list_t *)PDC_free(tmp); - } - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&lock_list_mutex_g); -#endif - request_region = (region_list_t *)PDC_free(request_region); - - if (dirty_reg == 0) { - // Perform lock release function - PDC_Data_Server_region_release(&in.lock_release, &out); - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in.lock_release); - HG_Destroy(handle); - } - } - // write lock release with mapping case - // do data transfer if it is write lock release with mapping. - else { - request_region = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - PDC_region_transfer_t_to_list_t(&in.lock_release.region, request_region); - lock_obj = PDC_Server_get_obj_region(in.lock_release.obj_id); - target_obj = PDC_Server_get_obj_region(in.analysis.output_obj_id); -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&lock_list_mutex_g); -#endif - DL_FOREACH(lock_obj->region_lock_head, elt) - { - if (PDC_is_same_region_list(request_region, elt) == 1 && elt->obj_id == in.analysis.obj_id) { - dirty_reg = 1; - tmp = (region_list_t *)PDC_malloc(sizeof(region_list_t)); - DL_FOREACH(target_obj->region_buf_map_head, eltt) - { - PDC_region_transfer_t_to_list_t(&(eltt->remote_region_unit), tmp); - if (PDC_is_same_region_shape(tmp, in.analysis.output_type_extent, request_region, - in.analysis.type_extent) == 1) { - type_size = PDC_get_var_type_size(in.analysis.data_type); - eltt->local_data_type = in.analysis.data_type; - - // get remote object memory addr - // NOTE: In this scenario, the 'target' is going to be the - // region used for the analysis RESULT. For the analysis input, - // we need to allocate a buffer which will gather the data which - // currently resides on client and has been locked, updated, then - // unlocked to prompt the mapping data transfer. - - data_buf = PDC_Server_maybe_allocate_region_buf_ptr( - in.lock_release.obj_id, in.lock_release.region, type_size); - if (in.lock_release.region.ndim == 1) { - dims[0] = in.analysis.region.count[0] / type_size; - remote_count = 1; - data_ptrs_to = (void **)PDC_malloc(sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(sizeof(size_t)); - *data_ptrs_to = data_buf; - *data_size_to = eltt->local_region.count[0]; - } - - else if (in.lock_release.region.ndim == 2) { - /* dims can be set directly from local_region_nunit!! */ - dims[0] = in.analysis.region.count[0] / type_size; - dims[1] = in.analysis.region.count[1] / type_size; - remote_count = dims[0]; - data_ptrs_to = (void **)PDC_malloc(remote_count * sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(remote_count * sizeof(size_t)); - data_ptrs_to[0] = data_buf + type_size * dims[1] * - ((eltt->local_region.start[0] / type_size) + - (eltt->local_region.start[1] / type_size)); - data_size_to[0] = eltt->local_region.count[0]; - for (k = 1; k < remote_count; k++) { - data_ptrs_to[k] = data_ptrs_to[k - 1] + data_size_to[0]; - data_size_to[k] = data_size_to[0]; - } - } - else if (in.lock_release.region.ndim == 3) { - /* dims can be set directly from local_region_nunit!! */ - dims[0] = in.analysis.region.count[0] / type_size; - dims[1] = in.analysis.region.count[1] / type_size; - dims[2] = in.analysis.region.count[2] / type_size; - remote_count = dims[0]; - data_ptrs_to = (void **)PDC_malloc(remote_count * sizeof(void *)); - data_size_to = (size_t *)PDC_malloc(remote_count * sizeof(size_t)); - data_ptrs_to[0] = data_buf + type_size * dims[1] * - ((eltt->local_region.start[0] / type_size) + - (eltt->local_region.start[1] / type_size)); - data_size_to[0] = - eltt->local_region.count[2] * (eltt->local_region.count[1] / type_size); - for (k = 1; k < remote_count; k++) { - data_ptrs_to[k] = data_ptrs_to[k - 1] + data_size_to[0]; - data_size_to[k] = data_size_to[0]; - } - } - - /* Create a new block handle to read the data */ - hg_ret = - HG_Bulk_create(hg_info->hg_class, remote_count, data_ptrs_to, - (hg_size_t *)data_size_to, HG_BULK_READWRITE, &remote_bulk_handle); - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "=Could not create bulk data handle"); - } - data_ptrs_to = (void **)PDC_free(data_ptrs_to); - data_size_to = (size_t *)PDC_free(data_size_to); - - buf_map_bulk_args = (struct buf_map_analysis_and_release_bulk_args *)PDC_malloc( - sizeof(struct buf_map_analysis_and_release_bulk_args)); - if (buf_map_bulk_args == NULL) - PGOTO_ERROR(HG_OTHER_ERROR, "buf_map_bulk_args memory allocation failed"); - - // memset(buf_map_bulk_args, 0, sizeof(struct buf_map_release_bulk_args)); - memset(buf_map_bulk_args, 0, sizeof(struct buf_map_analysis_and_release_bulk_args)); - buf_map_bulk_args->handle = handle; - buf_map_bulk_args->data_buf = data_buf; - buf_map_bulk_args->in = in.analysis; - buf_map_bulk_args->remote_obj_id = eltt->remote_obj_id; - buf_map_bulk_args->remote_reg_id = eltt->remote_reg_id; - buf_map_bulk_args->remote_region = eltt->remote_region_unit; - buf_map_bulk_args->remote_client_id = eltt->remote_client_id; - buf_map_bulk_args->remote_bulk_handle = remote_bulk_handle; -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_init(&(buf_map_bulk_args->work_mutex)); - hg_thread_cond_init(&(buf_map_bulk_args->work_cond)); -#endif - /* Pull bulk data */ - size = HG_Bulk_get_size(eltt->local_bulk_handle); - if (size != HG_Bulk_get_size(remote_bulk_handle)) { - error = 1; - PGOTO_ERROR(HG_OTHER_ERROR, "Local and remote bulk size does not match"); - } - - hg_ret = HG_Bulk_transfer( - hg_info->context, analysis_and_region_release_bulk_transfer_cb, buf_map_bulk_args, - HG_BULK_PULL, eltt->local_addr, eltt->local_bulk_handle, 0, remote_bulk_handle, 0, - size, HG_OP_ID_IGNORE); - - if (hg_ret != HG_SUCCESS) { - error = 1; - PGOTO_ERROR(hg_ret, "buf map Could not read bulk data"); - } - } - } - tmp = (region_list_t *)PDC_free(tmp); - } - } -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&lock_list_mutex_g); -#endif - request_region = (region_list_t *)PDC_free(request_region); - - if (dirty_reg == 0) { - // Perform lock release function - PDC_Data_Server_region_release(&in.lock_release, &out); - - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in.lock_release); - HG_Destroy(handle); - } - } - -done: - if (error == 1) { - out.ret = 0; - HG_Respond(handle, NULL, NULL, &out); - HG_Free_input(handle, &in); - HG_Destroy(handle); - } - - FUNC_LEAVE(ret_value); -} - // region_lock_cb HG_TEST_RPC_CB(region_lock, handle) { @@ -6110,9 +4919,6 @@ HG_TEST_THREAD_CB(update_region_loc) HG_TEST_THREAD_CB(get_metadata_by_id) HG_TEST_THREAD_CB(aggregate_write) HG_TEST_THREAD_CB(region_release) -HG_TEST_THREAD_CB(transform_region_release) -HG_TEST_THREAD_CB(region_analysis_release) -HG_TEST_THREAD_CB(region_transform_release) HG_TEST_THREAD_CB(metadata_add_tag) HG_TEST_THREAD_CB(metadata_add_kvtag) HG_TEST_THREAD_CB(metadata_get_kvtag) @@ -6183,10 +4989,7 @@ PDC_FUNC_DECLARE_REGISTER(buf_unmap) PDC_FUNC_DECLARE_REGISTER(region_lock) PDC_FUNC_DECLARE_REGISTER_IN_OUT(region_release, region_lock_in_t, region_lock_out_t) -PDC_FUNC_DECLARE_REGISTER_IN_OUT(transform_region_release, region_transform_and_lock_in_t, region_lock_out_t) -PDC_FUNC_DECLARE_REGISTER_IN_OUT(region_transform_release, region_transform_and_lock_in_t, region_lock_out_t) -PDC_FUNC_DECLARE_REGISTER_IN_OUT(region_analysis_release, region_analysis_and_lock_in_t, region_lock_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(query_partial, metadata_query_transfer_in_t, metadata_query_transfer_out_t) PDC_FUNC_DECLARE_REGISTER_IN_OUT(query_kvtag, pdc_kvtag_t, metadata_query_transfer_out_t) PDC_FUNC_DECLARE_REGISTER(bulk_rpc) diff --git a/src/server/pdc_server.c b/src/server/pdc_server.c index abe35434d..9fe4bcd11 100644 --- a/src/server/pdc_server.c +++ b/src/server/pdc_server.c @@ -48,9 +48,7 @@ #include "pdc_utlist.h" #include "pdc_hash_table.h" #include "pdc_interface.h" -#include "pdc_analysis_pkg.h" #include "pdc_client_server_common.h" -#include "pdc_transforms_common.h" #include "pdc_server.h" #include "pdc_server_metadata.h" #include "pdc_server_data.h" @@ -59,6 +57,9 @@ #include "pdc_server_region_transfer_metadata_query.h" #include "pdc_logger.h" #include "pdc_malloc.h" +#include "pdc_tf_server.h" +#include "pdc_tf_profiler.h" +#include "pdc_mercury_auth.h" #ifdef PDC_HAS_CRAY_DRC #include @@ -170,6 +171,97 @@ pdc_data_server_io_list_t * pdc_data_server_write_list_head_g = NULL; update_storage_meta_list_t * pdc_update_storage_meta_list_head_g = NULL; extern data_server_region_t *dataserver_region_g; +static pbool_t +PDC_is_cxi_mercury_transport(const char *hg_transport) +{ + return (hg_transport != NULL && + (strcmp(hg_transport, "ofi+cxi") == 0 || strcmp(hg_transport, "cxi") == 0)); +} + +static pbool_t +PDC_is_perlmutter_system(void) +{ + const char *nersc_host = getenv("NERSC_HOST"); + + return (nersc_host != NULL && strcmp(nersc_host, "perlmutter") == 0); +} + +static int +PDC_get_local_process_rank(int fallback_rank) +{ + const char *env_names[] = {"SLURM_LOCALID", "MPI_LOCALRANKID", "PMI_LOCAL_RANK", + "OMPI_COMM_WORLD_LOCAL_RANK"}; + size_t i; + + for (i = 0; i < sizeof(env_names) / sizeof(env_names[0]); i++) { + char *endptr = NULL; + char *env_value = getenv(env_names[i]); + long local_rank = 0; + + if (env_value == NULL || env_value[0] == '\0') + continue; + + local_rank = strtol(env_value, &endptr, 10); + if (endptr != env_value && *endptr == '\0' && local_rank >= 0 && local_rank <= INT_MAX) + return (int)local_rank; + } + + return fallback_rank; +} + +static void +PDC_get_default_cxi_endpoint(char *host_buf, size_t host_buf_len, int fallback_rank, int pid_base) +{ + const char *nic_name = getenv("HG_CXI_NIC"); + int local_rank = PDC_get_local_process_rank(fallback_rank); + int pid_offset = local_rank % 255; + int pid = pid_base + pid_offset; + + if (nic_name == NULL || nic_name[0] == '\0') + nic_name = "cxi0"; + + snprintf(host_buf, host_buf_len, "%s:%d", nic_name, pid); +} + +static const char * +PDC_get_default_mercury_transport(void) +{ + return (PDC_is_perlmutter_system() == TRUE) ? "ofi+cxi" : "ofi+tcp"; +} + +static perr_t +PDC_build_mercury_na_info_string(const char *hg_transport, int port, int cxi_pid_base, char *na_info_string, + size_t na_info_string_len, char *host_buf, size_t host_buf_len) +{ + perr_t ret_value = SUCCEED; + const char *hostname = getenv("HG_HOST"); + + if (hostname == NULL || hostname[0] == '\0') { + if (PDC_is_cxi_mercury_transport(hg_transport) == TRUE) { + PDC_get_default_cxi_endpoint(host_buf, host_buf_len, pdc_server_rank_g, cxi_pid_base); + hostname = host_buf; + if (pdc_server_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); + } + else { + memset(host_buf, 0, host_buf_len); + gethostname(host_buf, host_buf_len - 1); + hostname = host_buf; + if (pdc_server_rank_g == 0) + LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); + } + } + else + LOG_INFO("Environment variable HG_HOST was set\n"); + + if (PDC_is_cxi_mercury_transport(hg_transport) == TRUE) + snprintf(na_info_string, na_info_string_len, "%s://%s", hg_transport, hostname); + else + snprintf(na_info_string, na_info_string_len, "%s://%s:%d", hg_transport, hostname, port); + + return ret_value; +} + /* * Init the remote server info structure * @@ -743,9 +835,8 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) int i = 0; char self_addr_string[ADDR_MAX]; char na_info_string[NA_STRING_INFO_LEN]; - char * hostname; - pbool_t free_hostname = false; - struct hg_init_info init_info = {0}; + char host_buf[HOSTNAME_LEN]; + struct hg_init_info init_info = {0}; /* Set the default mercury transport * but enable overriding that to any of: @@ -753,8 +844,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) * "ofi+tcp" * "cci+tcp" */ - char *default_hg_transport = "ofi+tcp"; - char *hg_transport; + const char *default_hg_transport = PDC_get_default_mercury_transport(); + char * hg_transport; + int cxi_pid_base = 128; #ifdef PDC_HAS_CRAY_DRC uint32_t credential = 0, cookie; drc_info_handle_t credential_info; @@ -774,28 +866,20 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) total_mem_usage_g += (sizeof(char) + sizeof(char *)); if ((hg_transport = getenv("HG_TRANSPORT")) == NULL) { - hg_transport = default_hg_transport; + hg_transport = (char *)default_hg_transport; if (pdc_server_rank_g == 0) LOG_INFO("Environment variable HG_TRANSPORT was NOT set, default to %s\n", hg_transport); } else LOG_INFO("Environment variable HG_TRANSPORT was set\n"); - if ((hostname = getenv("HG_HOST")) == NULL) { - hostname = PDC_malloc(HOSTNAME_LEN); - memset(hostname, 0, HOSTNAME_LEN); - gethostname(hostname, HOSTNAME_LEN - 1); - free_hostname = true; - if (pdc_server_rank_g == 0) - LOG_INFO("Environment variable HG_HOST was NOT set, default to %s\n", hostname); - } - else - LOG_INFO("Environment variable HG_HOST was set\n"); - snprintf(na_info_string, NA_STRING_INFO_LEN, "%s://%s:%d", hg_transport, hostname, port); + + ret_value = PDC_build_mercury_na_info_string(hg_transport, port, cxi_pid_base, na_info_string, + sizeof(na_info_string), host_buf, sizeof(host_buf)); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Error building Mercury connection string"); if (pdc_server_rank_g == 0) LOG_INFO("Connection string: %s\n", na_info_string); - if (free_hostname) - hostname = PDC_free(hostname); // Clean up all the tmp files etc HG_Cleanup(); @@ -834,7 +918,25 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) #ifdef PDC_HAS_SHARED_SERVER init_info.auto_sm = HG_TRUE; #endif - *hg_class = HG_Init_opt(na_info_string, HG_TRUE, &init_info); + { + pdc_scoped_env_entry_t scoped_envs[PDC_PERLMUTTER_CXI_ENV_COUNT]; + int scoped_env_count = 0; + unsigned int scoped_svc_id = 0; + unsigned int scoped_vni = 0; + char scoped_device[32]; + perr_t scoped_auth_ret; + + scoped_auth_ret = + PDC_scope_perlmutter_cxi_auth_env(hg_transport, scoped_envs, &scoped_env_count, &scoped_svc_id, + &scoped_vni, scoped_device, sizeof(scoped_device)); + if (scoped_auth_ret != SUCCEED && pdc_server_rank_g == 0) + LOG_WARNING("Unable to scope Slingshot CXI auth for Mercury, falling back to provider defaults"); + else if (scoped_env_count > 0 && pdc_server_rank_g == 0) + LOG_INFO("Scoped Slingshot CXI auth for Mercury: svc=%u vni=%u device=%s\n", scoped_svc_id, + scoped_vni, scoped_device); + *hg_class = HG_Init_opt(na_info_string, HG_TRUE, &init_info); + PDC_restore_scoped_env(scoped_envs, scoped_env_count); + } if (*hg_class == NULL) { LOG_ERROR("Error with HG_Init()\n"); FUNC_LEAVE(FAIL); @@ -889,7 +991,7 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) hg_thread_pool_init(n_thread, &hg_test_thread_pool_g); hg_thread_pool_init(1, &hg_test_thread_pool_fs_g); if (pdc_server_rank_g == 0) - LOG_INFO("\nStarting server with %d threads...\n", n_thread); + LOG_INFO("Starting server with %d threads...\n", n_thread); hg_thread_mutex_init(&hash_table_new_mutex_g); hg_thread_mutex_init(&pdc_client_info_mutex_g); hg_thread_mutex_init(&pdc_metadata_hash_table_mutex_g); @@ -972,6 +1074,9 @@ PDC_Server_init(int port, hg_class_t **hg_class, hg_context_t **hg_context) PDC_region_server_cache_init(); #endif + if (PDCtf_init_builtin_funcs() != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDCtf_init_builtin_funcs"); + done: FUNC_LEAVE(ret_value); } @@ -1121,7 +1226,6 @@ PDC_Server_finalize() hg_thread_mutex_destroy(&update_remote_server_addr_mutex_g); #endif PDC_Server_clear_obj_region(); - PDC_server_transfer_request_finalize(); if (pdc_server_rank_g == 0) @@ -1180,6 +1284,9 @@ PDC_Server_checkpoint() { FUNC_ENTER(NULL); + if (pdc_server_rank_g == 0) + LOG_WARNING("Checkpoint start\n"); + perr_t ret_value = SUCCEED; pdc_metadata_t * elt; region_list_t * region_elt; @@ -1197,14 +1304,14 @@ PDC_Server_checkpoint() bool use_tmpfs = false; FILE * file; -#ifdef PDC_TIMING - // Timing + // #ifdef PDC_TIMING + // Timing struct timeval pdc_timer_start; struct timeval pdc_timer_end; struct timeval pdc_timer_end_rank; double checkpoint_time, checkpoint_time_rank; gettimeofday(&pdc_timer_start, 0); -#endif + // #endif env_char = getenv("PDC_CHECKPOINT_TMPFS"); if (env_char && atoi(env_char) != 0) @@ -1311,7 +1418,7 @@ PDC_Server_checkpoint() } // End for metadata entry linked list } // End for hash table metadata entry - // Note data server region are managed by data server instead of metadata server + // Note data server region is managed by data server instead of metadata server data_server_region_t *region = NULL; DL_COUNT(dataserver_region_g, region, n_objs); fwrite(&n_objs, sizeof(int), 1, file); @@ -1330,6 +1437,230 @@ PDC_Server_checkpoint() fwrite(&checkpoint_size, sizeof(uint64_t), 1, file); fwrite(checkpoint, checkpoint_size, 1, file); + /** + * Directed graph checkpoint format + * num_objs + * foreach obj: + * json_filepath_str + * num_region_mappings + * foreach region + * dg_id + * cur_state_str + * client_state_str + * store_state_str + * + * conceptual_region_ndim + * pdc_var_type + * conceptual_region_offset[DIM_MAX] + * conceptual_region_size + * + * actual_region_ndim + * pdc_var_type + * actual_region_size[DIM_MAX] + * + * foreach state_param + * state_id + * name + * conceptual_flat_offset + * state_param_size + * state_param_data + * + * foreach func_param + * func_id + * name + * params_str + * conceptual_flat_offset + * func_param_size + * func_param_data + */ + + // FIXME: We don't store whether graph is attached to entire object... + + if (pdc_server_rank_g == 0) + LOG_WARNING("Writing checkpoint transformations start\n"); + // Checkpoint the region transformations +#define PRINT_DEBUG_TRANSFORMATION_CHECKPOINTING +#ifdef PRINT_DEBUG_TRANSFORMATION_CHECKPOINTING + LOG_DEBUG("Checkpointing transformations\n"); + size_t num_objs = pdc_vector_size(tf_obj_id_to_dg_vector_g); + LOG_DEBUG("num_objs: %lu\n", num_objs); + PDC_VECTOR_ITERATOR *obj_id_to_dg_iter = pdc_vector_iterator_new(tf_obj_id_to_dg_vector_g); + while (pdc_vector_iterator_has_next(obj_id_to_dg_iter)) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = pdc_vector_iterator_next(obj_id_to_dg_iter); + LOG_DEBUG("obj_id: %d\n", cur_obj_id_to_dg->obj_id); + LOG_DEBUG("\tobj[%d] json_filepath_str: %s\n", cur_obj_id_to_dg->obj_id, + (char *)cur_obj_id_to_dg->dg->data); + + // Checkpoint region mapping + LOG_DEBUG("\tnum_region_mappings: %lu\n", + pdc_vector_size(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector)); + PDC_VECTOR_ITERATOR *region_mapping_iter = + pdc_vector_iterator_new(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector); + while (pdc_vector_iterator_has_next(region_mapping_iter)) { + pdc_tf_region_mapping_t *cur_region_mapping = pdc_vector_iterator_next(region_mapping_iter); + LOG_DEBUG("\t\tdg_id: %d\n", cur_region_mapping->region_state.dg_id); + + LOG_DEBUG("\t\tcur_state_str: %s\n", cur_region_mapping->region_state.cur_state); + LOG_DEBUG("\t\tclient_state_str: %s\n", cur_region_mapping->region_state.client_state); + LOG_DEBUG("\t\tstore_state_str: %s\n", cur_region_mapping->region_state.store_state); + + LOG_DEBUG("\t\tconceptual_region_ndim: %d\n", cur_region_mapping->conceptual_region.ndim); + LOG_DEBUG("\t\tpdc_var_type: %d, size: %d\n", cur_region_mapping->conceptual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->conceptual_region.pdc_var_type)); + LOG_DEBUG("\t\tconceptual_region_offset:\n"); + for (int i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + LOG_DEBUG("\t\t\toffset[%d]=%lu\n", i, cur_region_mapping->conceptual_offset[i]); + } + LOG_DEBUG("\t\tconceptual_region_size:\n"); + for (int i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + LOG_DEBUG("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->conceptual_region.size[i]); + } + + LOG_DEBUG("\t\tactual_region_ndim: %d\n", cur_region_mapping->actual_region.ndim); + LOG_DEBUG("\t\tpdc_var_type %d, size: %d\n", cur_region_mapping->actual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->actual_region.pdc_var_type)); + LOG_DEBUG("\t\tactual_region_size:\n"); + for (int i = 0; i < cur_region_mapping->actual_region.ndim; i++) { + LOG_DEBUG("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->actual_region.size[i]); + } + } + pdc_vector_iterator_destroy(region_mapping_iter); + + // Checkpoint state and func params for dg + for (int i = 0; i < cur_obj_id_to_dg->dg->edge_count; i++) { + pdc_tf_func_t *f = cur_obj_id_to_dg->dg->edges[i]->data; + LOG_DEBUG("\t\tfunc_name: %s\n", f->name); + LOG_DEBUG("\t\t\tparams_str: %s\n", + (f->params_str && strlen(f->params_str) > 0) ? f->params_str : "none"); + LOG_DEBUG("\t\t\tnum_params: %d\n", pdc_vector_size(f->pdc_tf_dg_params_vector)); + PDC_VECTOR_ITERATOR *cur_param_iter = pdc_vector_iterator_new(f->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(cur_param_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(cur_param_iter); + LOG_DEBUG("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + LOG_DEBUG("\t\t\tparams_size: %d\n", cur_param->params_size); + } + pdc_vector_iterator_destroy(cur_param_iter); + } + + for (int i = 0; i < cur_obj_id_to_dg->dg->vertex_count; i++) { + pdc_tf_state_t *s = cur_obj_id_to_dg->dg->vertices[i]->data; + LOG_DEBUG("\t\tstate_name: %s\n", s->name); + LOG_DEBUG("\t\t\tnum_params: %d\n", pdc_vector_size(s->pdc_tf_dg_params_vector)); + PDC_VECTOR_ITERATOR *cur_param_iter = pdc_vector_iterator_new(s->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(cur_param_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(cur_param_iter); + LOG_DEBUG("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + LOG_DEBUG("\t\t\tparams_size: %d\n", cur_param->params_size); + } + pdc_vector_iterator_destroy(cur_param_iter); + } + } + pdc_vector_iterator_destroy(obj_id_to_dg_iter); +#endif + + num_objs = pdc_vector_size(tf_obj_id_to_dg_vector_g); + fwrite(&num_objs, sizeof(size_t), 1, file); + + obj_id_to_dg_iter = pdc_vector_iterator_new(tf_obj_id_to_dg_vector_g); + while (pdc_vector_iterator_has_next(obj_id_to_dg_iter)) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = pdc_vector_iterator_next(obj_id_to_dg_iter); + // Write object ID + fwrite(&cur_obj_id_to_dg->obj_id, sizeof(pdcid_t), 1, file); + // Write JSON filepath string length and data + size_t json_path_len = strlen((char *)cur_obj_id_to_dg->dg->data) + 1; + fwrite(&json_path_len, sizeof(size_t), 1, file); + fwrite(cur_obj_id_to_dg->dg->data, sizeof(char), json_path_len, file); + + // Region mappings + size_t num_region_mappings = pdc_vector_size(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector); + fwrite(&num_region_mappings, sizeof(size_t), 1, file); + + PDC_VECTOR_ITERATOR *region_mapping_iter = + pdc_vector_iterator_new(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector); + while (pdc_vector_iterator_has_next(region_mapping_iter)) { + pdc_tf_region_mapping_t *cur_region_mapping = pdc_vector_iterator_next(region_mapping_iter); + + // Write dg_id + fwrite(&cur_region_mapping->region_state.dg_id, sizeof(pdcid_t), 1, file); + + // Write cur_state_str + size_t cur_state_len = strlen(cur_region_mapping->region_state.cur_state) + 1; + fwrite(&cur_state_len, sizeof(size_t), 1, file); + fwrite(cur_region_mapping->region_state.cur_state, sizeof(char), cur_state_len, file); + + // Write client_state_str + size_t client_state_len = strlen(cur_region_mapping->region_state.client_state) + 1; + fwrite(&client_state_len, sizeof(size_t), 1, file); + fwrite(cur_region_mapping->region_state.client_state, sizeof(char), client_state_len, file); + + // Write store_state_str + size_t store_state_len = strlen(cur_region_mapping->region_state.store_state) + 1; + fwrite(&store_state_len, sizeof(size_t), 1, file); + fwrite(cur_region_mapping->region_state.store_state, sizeof(char), store_state_len, file); + + // Write conceptual region + fwrite(&cur_region_mapping->conceptual_region.ndim, sizeof(size_t), 1, file); + fwrite(&cur_region_mapping->conceptual_region.pdc_var_type, sizeof(pdc_var_type_t), 1, file); + fwrite(cur_region_mapping->conceptual_region.size, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + + // Write conceptual offset + fwrite(cur_region_mapping->conceptual_offset, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + + // Write actual region + fwrite(&cur_region_mapping->actual_region.ndim, sizeof(size_t), 1, file); + fwrite(&cur_region_mapping->actual_region.pdc_var_type, sizeof(pdc_var_type_t), 1, file); + fwrite(cur_region_mapping->actual_region.size, sizeof(uint64_t), + cur_region_mapping->actual_region.ndim, file); + } + pdc_vector_iterator_destroy(region_mapping_iter); + + // Checkpoint function parameters + for (int i = 0; i < cur_obj_id_to_dg->dg->edge_count; i++) { + pdc_tf_func_t *f = cur_obj_id_to_dg->dg->edges[i]->data; + + // Write number of params + size_t num_params = pdc_vector_size(f->pdc_tf_dg_params_vector); + fwrite(&num_params, sizeof(size_t), 1, file); + PDC_VECTOR_ITERATOR *cur_param_iter = pdc_vector_iterator_new(f->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(cur_param_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(cur_param_iter); + + // Write conceptual_flat_offset and params_size; + fwrite(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fwrite(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Write param data + fwrite(cur_param->params, cur_param->params_size, 1, file); + } + pdc_vector_iterator_destroy(cur_param_iter); + } + + // Checkpoint state parameters + for (int i = 0; i < cur_obj_id_to_dg->dg->vertex_count; i++) { + pdc_tf_state_t *s = cur_obj_id_to_dg->dg->vertices[i]->data; + + // Write number of params + size_t num_params = pdc_vector_size(s->pdc_tf_dg_params_vector); + fwrite(&num_params, sizeof(size_t), 1, file); + + PDC_VECTOR_ITERATOR *cur_param_iter = pdc_vector_iterator_new(s->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(cur_param_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(cur_param_iter); + + // Write conceptual_flat_offset and params_size + fwrite(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fwrite(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Write param data + fwrite(cur_param->params, cur_param->params_size, 1, file); + } + pdc_vector_iterator_destroy(cur_param_iter); + } + } + pdc_vector_iterator_destroy(obj_id_to_dg_iter); + if (pdc_server_rank_g == 0) + LOG_WARNING("Writing checkpoint transformations done\n"); + fclose(file); if (use_tmpfs) { @@ -1343,10 +1674,10 @@ PDC_Server_checkpoint() system(cmd); } -#ifdef PDC_TIMING + // #ifdef PDC_TIMING gettimeofday(&pdc_timer_end_rank, 0); checkpoint_time_rank = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end_rank); -#endif + // #endif int all_metadata_size, all_region_count; #ifdef ENABLE_MPI @@ -1357,26 +1688,24 @@ PDC_Server_checkpoint() all_region_count = region_count; #endif -#ifdef PDC_TIMING - LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, - checkpoint_time_rank); + // #ifdef PDC_TIMING + // LOG_INFO("Checkpointed %10d objects, with %10d regions, took %7.2fs\n", metadata_size, region_count, + // checkpoint_time_rank); gettimeofday(&pdc_timer_end, 0); checkpoint_time = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); if (pdc_server_rank_g == 0) - LOG_ERROR("Rank[ ALL]: Total checkpoint time = %.6f\n", checkpoint_time); -#endif - - if (pdc_server_rank_g == 0) { - LOG_INFO("Checkpointed %10d objects, with %10d regions \n", all_metadata_size, all_region_count); - } + LOG_WARNING("Rank[ALL]: Total checkpoint time = %.6f\n", checkpoint_time); + // #endif #ifdef PDC_ENABLE_IDIOMS metadata_index_dump(pdc_server_tmp_dir_g, pdc_server_rank_g); #endif done: + if (pdc_server_rank_g == 0) + LOG_WARNING("Checkpointing done\n"); FUNC_LEAVE(ret_value); } // End Checkpoint @@ -1390,6 +1719,18 @@ region_cmp(region_list_t *a, region_list_t *b) FUNC_LEAVE(memcmp(a->start, b->start, unit_size)); } +static size_t read_checkpoint_str_len; + +#define READ_CHECKPOINT_STR(file, str_ptr) \ + do { \ + fread(&read_checkpoint_str_len, sizeof(size_t), 1, (file)); \ + LOG_DEBUG("read_checkpoint_str_len: %d\n", read_checkpoint_str_len); \ + if (read_checkpoint_str_len > 0) { \ + (str_ptr) = PDC_calloc(1, read_checkpoint_str_len); \ + fread((str_ptr), read_checkpoint_str_len, 1, (file)); \ + } \ + } while (0) + /* * Load metadata from checkpoint file in persistant storage * @@ -1402,6 +1743,9 @@ PDC_Server_restart(char *filename) { FUNC_ENTER(NULL); + if (pdc_server_rank_g == 0) + LOG_WARNING("Restart from checkpoint start\n"); + perr_t ret_value = SUCCEED; int n_entry, count, i, j, nobj = 0, all_nobj = 0, all_n_region, n_region, n_objs, total_region = 0, n_kvtag, key_len; @@ -1530,9 +1874,6 @@ PDC_Server_restart(char *filename) if (n_region < 0) PGOTO_ERROR(FAIL, "Checkpoint file region was less than 0"); - /* if (n_region == 0) */ - /* continue; */ - total_region += n_region; for (j = 0; j < n_region; j++) { @@ -1668,9 +2009,140 @@ PDC_Server_restart(char *filename) transfer_request_metadata_query_init(pdc_server_size_g, checkpoint_buf); checkpoint_buf = (char *)PDC_free(checkpoint_buf); + // FIXME: this has to go somehwere else... + PDCtf_init_builtin_funcs(); + + if (pdc_server_rank_g == 0) + LOG_WARNING("Reading checkpoint transformations start\n"); + LOG_DEBUG("Reading checkpoint transformations\n"); + size_t num_objs; + fread(&num_objs, sizeof(size_t), 1, file); + LOG_DEBUG("num_objs: %lu\n", num_objs); + tf_obj_id_to_dg_vector_g = pdc_vector_create(PDC_MAX(num_objs, 8), 2.0); + for (int _o = 0; _o < num_objs; _o++) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = PDC_calloc(1, sizeof(pdc_tf_obj_id_to_dg_t)); + pdc_vector_add(tf_obj_id_to_dg_vector_g, cur_obj_id_to_dg); + + fread(&cur_obj_id_to_dg->obj_id, sizeof(pdcid_t), 1, file); + LOG_DEBUG("obj_id: %d\n", cur_obj_id_to_dg->obj_id); + + char *json_filepath; + READ_CHECKPOINT_STR(file, json_filepath); + LOG_JUST_PRINT("\tobj[%d] json_filepath_str: %s\n", cur_obj_id_to_dg->obj_id, json_filepath); + + // Read checkpoint region mapping + size_t num_region_mappings; + fread(&num_region_mappings, sizeof(size_t), 1, file); + LOG_JUST_PRINT("\tnum_region_mappings: %lu\n", num_region_mappings); + cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector = + pdc_vector_create(PDC_MAX(num_region_mappings, 8), 2.0); + for (int _r = 0; _r < num_region_mappings; _r++) { + pdc_tf_region_mapping_t *cur_region_mapping = PDC_calloc(1, sizeof(pdc_tf_region_mapping_t)); + pdc_vector_add(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector, cur_region_mapping); + + pdcid_t dg_id; + fread(&dg_id, sizeof(pdcid_t), 1, file); + cur_region_mapping->region_state.dg_id = dg_id; + LOG_JUST_PRINT("\t\tdg_id: %d\n", cur_region_mapping->region_state.dg_id); + + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.cur_state); + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.client_state); + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.store_state); + + LOG_JUST_PRINT("\t\tcur_state_str: %s\n", cur_region_mapping->region_state.cur_state); + LOG_JUST_PRINT("\t\tclient_state_str: %s\n", cur_region_mapping->region_state.client_state); + LOG_JUST_PRINT("\t\tstore_state_str: %s\n", cur_region_mapping->region_state.store_state); + + fread(&(cur_region_mapping->conceptual_region.ndim), sizeof(size_t), 1, file); + fread(&(cur_region_mapping->conceptual_region.pdc_var_type), sizeof(pdc_var_type_t), 1, file); + fread(cur_region_mapping->conceptual_region.size, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + fread(cur_region_mapping->conceptual_offset, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + + LOG_JUST_PRINT("\t\tconceptual_region_ndim: %d\n", cur_region_mapping->conceptual_region.ndim); + LOG_JUST_PRINT("\t\tpdc_var_type: %d, size: %d\n", + cur_region_mapping->conceptual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->conceptual_region.pdc_var_type)); + LOG_JUST_PRINT("\t\tconceptual_region_offset:\n"); + for (i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + LOG_JUST_PRINT("\t\t\toffset[%d]=%lu\n", i, cur_region_mapping->conceptual_offset[i]); + } + LOG_JUST_PRINT("\t\tconceptual_region_size:\n"); + for (i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + LOG_JUST_PRINT("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->conceptual_region.size[i]); + } + + fread(&(cur_region_mapping->actual_region.ndim), sizeof(size_t), 1, file); + fread(&(cur_region_mapping->actual_region.pdc_var_type), sizeof(pdc_var_type_t), 1, file); + fread(cur_region_mapping->actual_region.size, sizeof(uint64_t), + cur_region_mapping->actual_region.ndim, file); + + LOG_JUST_PRINT("\t\tactual_region_ndim: %d\n", cur_region_mapping->actual_region.ndim); + LOG_JUST_PRINT("\t\tpdc_var_type %d, size: %d\n", cur_region_mapping->actual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->actual_region.pdc_var_type)); + LOG_JUST_PRINT("\t\tactual_region_size:\n"); + for (i = 0; i < cur_region_mapping->actual_region.ndim; i++) { + LOG_JUST_PRINT("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->actual_region.size[i]); + } + } + + cur_obj_id_to_dg->dg = PDCtf_dg_json_create_common(json_filepath); + + // Checkpoint state and func params for dg + for (int e_index = 0; e_index < cur_obj_id_to_dg->dg->edge_count; e_index++) { + pdc_tf_func_t *f = cur_obj_id_to_dg->dg->edges[e_index]->data; + size_t num_params; + fread(&num_params, sizeof(size_t), 1, file); + LOG_JUST_PRINT("\t\tfunc_name: %s\n", f->name); + LOG_JUST_PRINT("\t\t\tparams_str: %s\n", + (f->params_str && strlen(f->params_str) > 0) ? f->params_str : "none"); + LOG_JUST_PRINT("\t\t\tnum_params: %d\n", num_params); + f->pdc_tf_dg_params_vector = pdc_vector_create(PDC_MAX(num_params, 2), 2.0); + for (int _n = 0; _n < num_params; _n++) { + pdc_tf_dg_params_t *cur_param = PDC_calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(f->pdc_tf_dg_params_vector, cur_param); + + // Read conceptual_flat_offset and params_size + fread(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fread(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Read param data + cur_param->params = PDC_calloc(1, cur_param->params_size); + fread(cur_param->params, cur_param->params_size, 1, file); + + LOG_JUST_PRINT("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + LOG_JUST_PRINT("\t\t\tparams_size: %d\n", cur_param->params_size); + } + } + + for (int v_index = 0; v_index < cur_obj_id_to_dg->dg->vertex_count; v_index++) { + pdc_tf_state_t *s = cur_obj_id_to_dg->dg->vertices[v_index]->data; + size_t num_params; + fread(&num_params, sizeof(size_t), 1, file); + LOG_JUST_PRINT("\t\tstate_name: %s\n", s->name); + LOG_JUST_PRINT("\t\t\tnum_params: %d\n", num_params); + s->pdc_tf_dg_params_vector = pdc_vector_create(PDC_MAX(num_params, 2), 2.0); + for (int _n = 0; _n < num_params; _n++) { + pdc_tf_dg_params_t *cur_param = PDC_calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(s->pdc_tf_dg_params_vector, cur_param); + + // Read conceptual_flat_offset and params_size + fread(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fread(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Read param data + cur_param->params = PDC_calloc(1, cur_param->params_size); + fread(cur_param->params, cur_param->params_size, 1, file); + + LOG_JUST_PRINT("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + LOG_JUST_PRINT("\t\t\tparams_size: %d\n", cur_param->params_size); + } + } + } + if (pdc_server_rank_g == 0) + LOG_WARNING("Reading checkpoint transformations done\n"); + fclose(file); file = NULL; - #ifdef ENABLE_MPI MPI_Reduce(&nobj, &all_nobj, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(&total_region, &all_n_region, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); @@ -1678,7 +2150,6 @@ PDC_Server_restart(char *filename) all_nobj = nobj; all_n_region = total_region; #endif - if (pdc_server_rank_g == 0) { LOG_INFO("Server restarted from saved session, " "successfully loaded %d containers, %d objects, %d regions...\n", @@ -1689,7 +2160,8 @@ PDC_Server_restart(char *filename) #ifdef PDC_TIMING pdc_server_timings->PDCserver_restart += MPI_Wtime() - start; #endif - + if (pdc_server_rank_g == 0) + LOG_WARNING("Restart from checkpoint done\n"); FUNC_LEAVE(ret_value); } @@ -1766,9 +2238,11 @@ PDC_Server_loop(hg_context_t *hg_context) { FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - hg_return_t hg_ret; - unsigned int actual_count; + perr_t ret_value = SUCCEED; + hg_return_t hg_ret; + unsigned int actual_count; + struct timespec hg_progress_start, hg_progress_end; + struct timespec total_time_start, total_time_end; #ifdef PDC_ENABLE_CHECKPOINT int checkpoint_interval = 1; clock_t last_checkpoint_time = 0, cur_time; @@ -1776,6 +2250,7 @@ PDC_Server_loop(hg_context_t *hg_context) /* Poke progress engine and check for events */ do { + clock_gettime(CLOCK_MONOTONIC, &total_time_start); #ifdef PDC_ENABLE_CHECKPOINT checkpoint_interval++; // Avoid calling clock() every operation @@ -1796,11 +2271,24 @@ PDC_Server_loop(hg_context_t *hg_context) hg_ret = HG_Trigger(hg_context, 0 /* timeout */, 1 /* max count */, &actual_count); } while ((hg_ret == HG_SUCCESS) && actual_count); - /* Do not try to make progress anymore if we're done */ if (hg_atomic_cas32(&close_server_g, 1, 1)) break; + clock_gettime(CLOCK_MONOTONIC, &hg_progress_start); hg_ret = HG_Progress(hg_context, 200); - + clock_gettime(CLOCK_MONOTONIC, &hg_progress_end); + clock_gettime(CLOCK_MONOTONIC, &total_time_end); + + double elapsed_total_time_sec = (total_time_end.tv_sec - total_time_start.tv_sec) + + (total_time_end.tv_nsec - total_time_start.tv_nsec) / 1e9; + // NOTE: this must be called after HG_Progress + double elapsed_progress_time_sec = (hg_progress_end.tv_sec - hg_progress_start.tv_sec) + + (hg_progress_end.tv_nsec - hg_progress_start.tv_nsec) / 1e9; + // NOTE: this must be called after HG_Progress + /*if (pdc_tf_update_profiler(elapsed_total_time_sec, elapsed_progress_time_sec) != SUCCEED) { + LOG_ERROR("Error updating profiler\n"); + ret_value = FAIL; + break; + }*/ } while (hg_ret == HG_SUCCESS || hg_ret == HG_TIMEOUT); if (hg_ret == HG_SUCCESS) @@ -1884,7 +2372,6 @@ PDC_print_IO_stats() update_time_avg = update_time_max = update_time_min = server_update_region_location_time_g; get_info_time_avg = get_info_time_max = get_info_time_min = server_get_storage_info_time_g; io_elapsed_time_avg = io_elapsed_time_max = io_elapsed_time_min = server_io_elapsed_time_g; - #endif if (pdc_server_rank_g == 0) { @@ -1966,15 +2453,6 @@ PDC_Server_mercury_register() PDC_send_data_query_rpc_register(hg_class_g); PDC_get_sel_data_rpc_register(hg_class_g); - // Analysis and Transforms - PDC_set_execution_locus(SERVER_MEMORY); - PDC_obj_data_iterator_register(hg_class_g); - PDC_analysis_ftn_register(hg_class_g); - PDC_transform_ftn_register(hg_class_g); - PDC_transform_region_release_register(hg_class_g); - PDC_region_transform_release_register(hg_class_g); - PDC_region_analysis_release_register(hg_class_g); - // DART Index #ifdef PDC_ENABLE_IDIOMS PDC_dart_get_server_info_register(hg_class_g); @@ -2156,8 +2634,14 @@ server_run(int argc, char *argv[]) PDC_server_timing_init(); #endif #endif - if (argc > 1 && strcmp(argv[1], "restart") == 0) + if (argc > 1 && strcmp(argv[1], "restart") == 0) { + if (pdc_server_rank_g == 0) + LOG_INFO("Starting server with pre-existing data\n"); is_restart_g = 1; + } + else if (pdc_server_rank_g == 0) { + LOG_INFO("Starting server with no previous data\n"); + } // Init rand seed srand(time(NULL)); @@ -2189,6 +2673,8 @@ server_run(int argc, char *argv[]) PDC_Server_write_addr_to_file(all_addr_strings_g, pdc_server_size_g) != SUCCEED) { LOG_ERROR("Error with write config file\n"); } + else if (pdc_server_rank_g == 0) + LOG_INFO("Succesfully wrote address to config in dir %s\n", pdc_server_tmp_dir_g); #ifdef ENABLE_ROCKSDB if (use_rocksdb_g) { diff --git a/src/server/pdc_server_analysis/include/pdc_server_analysis.h b/src/server/pdc_server_analysis/include/pdc_server_analysis.h deleted file mode 100644 index 8b6e3134b..000000000 --- a/src/server/pdc_server_analysis/include/pdc_server_analysis.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#ifndef PDC_ANALYSIS_SERVER_H -#define PDC_ANALYSIS_SERVER_H - -#include "pdc_public.h" -#include "pdc_analysis_and_transforms_common.h" - -perr_t PDC_Server_instantiate_data_iterator(obj_data_iterator_in_t *in, obj_data_iterator_out_t *out); - -int PDC_get_analysis_registry(struct _pdc_region_analysis_ftn_info ***registry); - -#endif diff --git a/src/server/pdc_server_analysis/pdc_server_analysis.c b/src/server/pdc_server_analysis/pdc_server_analysis.c deleted file mode 100644 index ae6a98632..000000000 --- a/src/server/pdc_server_analysis/pdc_server_analysis.c +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "mercury.h" -#include "mercury_macros.h" - -// Mercury hash table and list -#include "mercury_hash_table.h" -#include "mercury_list.h" - -#include "pdc_config.h" - -#ifdef ENABLE_MPI -#include "mpi.h" -#endif - -#include "pdc_utlist.h" -#include "pdc_dablooms.h" -#include "pdc_interface.h" -#include "pdc_client_server_common.h" -#include "pdc_server.h" -#include "pdc_analysis_pkg.h" -#include "pdc_analysis.h" -#include "pdc_logger.h" -#include "pdc_timing.h" - -#ifdef PDC_HAS_CRAY_DRC -#include -#endif - -#define BLOOM_TYPE_T counting_bloom_t -#define BLOOM_NEW new_counting_bloom -#define BLOOM_CHECK counting_bloom_check -#define BLOOM_ADD counting_bloom_add -#define BLOOM_REMOVE counting_bloom_remove -#define BLOOM_FREE free_counting_bloom - -#ifdef ENABLE_MULTITHREAD -hg_thread_mutex_t insert_iterator_mutex_g = HG_THREAD_MUTEX_INITIALIZER; -#endif - -/* - * Insert an iterator received from client into a collection - * - * \param in[IN] Input structure received from client - * \param out[OUT] Output structure to be sent back to the client - * - * \return Non-negative on success/Negative on failure - */ - -perr_t -PDC_Server_instantiate_data_iterator(obj_data_iterator_in_t *in, obj_data_iterator_out_t *out) -{ - FUNC_ENTER(NULL); - - perr_t ret_value = SUCCEED; - data_server_region_t * region_reference = NULL; - struct _pdc_iterator_info *thisIter; - -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_lock(&insert_iterator_mutex_g); -#endif - int nextId = PDCiter_get_nextId(); - thisIter = &PDC_Block_iterator_cache[nextId]; -#ifdef ENABLE_MULTITHREAD - hg_thread_mutex_unlock(&insert_iterator_mutex_g); -#endif - thisIter->objectId = in->object_id; - thisIter->reg_id = in->reg_id; - thisIter->sliceCount = in->sliceCount; - thisIter->sliceNext = in->sliceNext; - thisIter->sliceResetCount = in->sliceResetCount; - thisIter->elementsPerSlice = in->elementsPerSlice; - thisIter->slicePerBlock = in->slicePerBlock; - thisIter->elementsPerPlane = in->elementsPerPlane; - thisIter->elementsPerBlock = in->elementsPerBlock; - thisIter->skipCount = in->skipCount; - thisIter->element_size = in->element_size; - thisIter->srcBlockCount = in->srcBlockCount; - thisIter->contigBlockSize = in->contigBlockSize; - thisIter->totalElements = in->totalElements; - thisIter->pdc_datatype = (pdc_var_type_t)(in->storageinfo & 0x0FF); - thisIter->storage_order = (_pdc_major_type_t)((in->storageinfo >> 8) & 0xFF); - region_reference = PDC_Server_get_obj_region(in->object_id); - if (region_reference == NULL) { - LOG_ERROR("Unable to locate object region (id=%" PRIu64 ")\n", in->object_id); - /* The most likely cause of this condition is that the client never - * created an object mapping which would move the client data to the data-server. - * We now have the option to either fail, or to create a new temporary region. - * With the latter, we can indicate in our return that we've created the - * data-server temp so that the client can update the locus info. - */ - out->server_region_id = -1; - } - else { - LOG_INFO("Found object region for id=%" PRIu64 "\n", in->object_id); - out->server_region_id = in->object_id; - } - - out->client_iter_id = in->client_iter_id; - out->server_iter_id = nextId; - out->ret = 0; - - FUNC_LEAVE(ret_value); -} - -/* - * Insert an iterator received from client into a collection - * - * \param in[IN] Input structure received from client - * \param out[OUT] Output structure to be sent back to the client - * - * \return Non-negative on success/Negative on failure - */ - -void * -PDC_Server_get_region_data_ptr(pdcid_t object_id) -{ - FUNC_ENTER(NULL); - - data_server_region_t *region_reference = NULL; - region_reference = PDC_Server_get_obj_region(object_id); - if (region_reference) { - /* See if this is a mapped object/region */ - if (region_reference->region_buf_map_head) { - return region_reference->region_buf_map_head->remote_data_ptr; - } - else - return region_reference->obj_data_ptr; - } - - FUNC_LEAVE(NULL); -} - -void * -PDC_Server_get_ftn_reference(char *ftn) -{ - FUNC_ENTER(NULL); - - static void *appHandle = NULL; - void * ftnHandle = NULL; - if (appHandle == NULL) { - /* We need the in_process address of the function */ - if ((appHandle = dlopen(0, RTLD_NOW)) == NULL) { - char *this_error = dlerror(); - LOG_ERROR("dlopen failed: %s\n", this_error); - FUNC_LEAVE(NULL); - } - } - ftnHandle = dlsym(appHandle, ftn); - - FUNC_LEAVE(ftnHandle); -} - -size_t -PDCobj_data_getSliceCount(pdcid_t iter) -{ - FUNC_ENTER(NULL); - - struct _pdc_iterator_info *thisIter = NULL; - /* Special case to handle a NULL iterator */ - if (iter == 0) - FUNC_LEAVE(0); - /* FIXME: Should add another check to see that the input - * iter id is in the range of cached values... - */ - if ((PDC_Block_iterator_cache != NULL) && (iter > 0)) { - thisIter = &PDC_Block_iterator_cache[iter]; - FUNC_LEAVE(thisIter->sliceCount); - } - - FUNC_LEAVE(0); -} - -size_t -PDCobj_data_getNextBlock(pdcid_t iter, void **nextBlock, size_t *dims) -{ - FUNC_ENTER(NULL); - int ret_value = SUCCEED; - - struct _pdc_iterator_info *thisIter = NULL; - /* Special case to handle a NULL iterator */ - if (iter == 0) { - if (nextBlock != NULL) - *nextBlock = NULL; - if (dims != NULL) - *dims = 0; - - FUNC_LEAVE(0); - } - - if ((PDC_Block_iterator_cache != NULL) && (iter > 0)) { - thisIter = &PDC_Block_iterator_cache[iter]; - if (thisIter->srcStart == NULL) { - if (execution_locus == SERVER_MEMORY) { - if ((thisIter->srcNext = PDC_Server_get_region_data_ptr(thisIter->objectId)) == NULL) - thisIter->srcNext = PDC_malloc(thisIter->totalElements * thisIter->element_size); - if ((thisIter->srcStart = thisIter->srcNext) == NULL) { - LOG_ERROR("Unable to allocate iterator storage\n"); - FUNC_LEAVE(0); - } - thisIter->srcNext += thisIter->startOffset + thisIter->skipCount; - } - } - if (thisIter->srcNext != NULL) { - if (thisIter->sliceNext == thisIter->sliceCount) { - /* May need to adjust the elements in this last - * block... - */ - size_t current_total = thisIter->sliceCount * thisIter->elementsPerBlock; - size_t remaining = 0; - - if (current_total == thisIter->totalElements) { - if (nextBlock) - *nextBlock = NULL; - thisIter->sliceNext = 0; - thisIter->srcNext = NULL; - PGOTO_DONE(ret_value); - } - if (nextBlock) - *nextBlock = thisIter->srcNext; - thisIter->srcNext = NULL; - remaining = thisIter->totalElements - current_total; - if (dims) { - if (thisIter->storage_order == ROW_major) - dims[0] = remaining / thisIter->elementsPerSlice; - else - dims[1] = remaining / thisIter->elementsPerSlice; - } - return remaining; - } - else if (thisIter->sliceNext && (thisIter->sliceNext % thisIter->sliceResetCount) == 0) { - size_t offset = ++thisIter->srcBlockCount * thisIter->elementsPerBlock; - thisIter->srcNext = thisIter->srcStart + offset + thisIter->skipCount; - if (nextBlock) - *nextBlock = thisIter->srcNext; - } - else { - *nextBlock = thisIter->srcNext; - thisIter->srcNext += thisIter->contigBlockSize; - } - thisIter->sliceNext += 1; - if (dims != NULL) { - dims[0] = thisIter->dims[0]; - if (thisIter->ndim > 1) - dims[1] = thisIter->dims[1]; - if (thisIter->ndim > 2) - dims[2] = thisIter->dims[2]; - if (thisIter->ndim > 3) - dims[2] = thisIter->dims[3]; - } - FUNC_LEAVE(thisIter->elementsPerBlock); - } - } - -done: - if (dims) - dims[0] = dims[1] = 0; - if (nextBlock) - *nextBlock = NULL; - - FUNC_LEAVE(ret_value); -} - -int -PDC_get_analysis_registry(struct _pdc_region_analysis_ftn_info ***registry) -{ - FUNC_ENTER(NULL); - - if (registry) { - *registry = pdc_region_analysis_registry; - FUNC_LEAVE(hg_atomic_get32(®istered_analysis_ftn_count_g)); - } - - FUNC_LEAVE(0); -} diff --git a/src/server/pdc_server_metadata.c b/src/server/pdc_server_metadata.c index efec78ff5..87bc1b256 100644 --- a/src/server/pdc_server_metadata.c +++ b/src/server/pdc_server_metadata.c @@ -209,12 +209,9 @@ PDC_Server_metadata_init(pdc_metadata_t *a) a->region_buf_map_head = NULL; a->prev = NULL; a->next = NULL; - a->transform_state = 0; - memset(&a->current_state, 0, sizeof(struct _pdc_transform_state)); FUNC_LEAVE_VOID(); } -// ^ hash table /* * Concatenate the metadata's obj_name and time_step to one char array @@ -803,31 +800,19 @@ PDC_Server_update_metadata(metadata_update_in_t *in, metadata_update_out_t *out) target->tags[strlen(target->tags)] = ','; strcat(target->tags, in->new_metadata.tags); } - if (in->new_metadata.current_state != 0) { - target->transform_state = in->new_metadata.current_state; - target->current_state.dtype = in->new_metadata.t_dtype; - target->current_state.ndim = in->new_metadata.t_ndim; - target->current_state.dims[0] = in->new_metadata.t_dims0; - target->current_state.dims[1] = in->new_metadata.t_dims1; - target->current_state.dims[2] = in->new_metadata.t_dims2; - target->current_state.dims[3] = in->new_metadata.t_dims3; - target->current_state.meta_index = in->new_metadata.t_meta_index; - } out->ret = 1; - } // if (lookup_value != NULL) + } else { // Object not found for deletion request ret_value = -1; out->ret = -1; } - - } // if lookup_value != NULL + } else { ret_value = -1; out->ret = -1; } - - } // if (metadata_hash_table_g != NULL) + } else { out->ret = -1; PGOTO_ERROR(FAIL, "metadata_hash_table_g not initialized"); @@ -840,7 +825,6 @@ PDC_Server_update_metadata(metadata_update_in_t *in, metadata_update_out_t *out) #endif #ifdef ENABLE_TIMING - // Timing gettimeofday(&pdc_timer_end, 0); ht_total_sec = PDC_get_elapsed_time_double(&pdc_timer_start, &pdc_timer_end); #endif @@ -1882,7 +1866,7 @@ PDC_Server_search_with_name_timestep(const char *obj_name, uint32_t hash_key, ui // lookup lookup_value = hash_table_lookup(metadata_hash_table_g, &hash_key); - // Is this hash value exist in the Hash table? + // Does this hash value exist in the Hash table? if (lookup_value != NULL) { *out = find_identical_metadata(lookup_value, &metadata); if (*out == NULL) diff --git a/src/server/pdc_server_metadata_index_test.c b/src/server/pdc_server_metadata_index_test.c index d1e664aa2..2ff036168 100644 --- a/src/server/pdc_server_metadata_index_test.c +++ b/src/server/pdc_server_metadata_index_test.c @@ -52,12 +52,11 @@ delete_kv_from_index(char *kv, uint64_t obj_id) #ifndef PDC_DART_SFX_TREE input.inserting_suffix = 0; input.attr_key = strdup(key); - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); input.inserting_suffix = 1; input.attr_key = reverse_str(key); - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); - + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); #else for (int i = 0; i < strlen(key); i++) { @@ -68,7 +67,7 @@ delete_kv_from_index(char *kv, uint64_t obj_id) input.inserting_suffix = 1; } input.attr_key = substring(key, i, strlen(key)); - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); } #endif @@ -123,12 +122,11 @@ insert_kv_to_index(char *kv, uint64_t obj_id) #ifndef PDC_DART_SFX_TREE input.inserting_suffix = 0; input.attr_key = strdup(key); - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); input.inserting_suffix = 1; input.attr_key = reverse_str(key); - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); - + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); #else for (int i = 0; i < strlen(key); i++) { @@ -140,7 +138,7 @@ insert_kv_to_index(char *kv, uint64_t obj_id) } input.attr_key = substring(key, i, strlen(key)); input.vnode_id = 1; - assert(PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL) == SUCCEED); + PDC_Server_dart_perform_one_server(&input, &output, NULL, NULL); } #endif @@ -161,7 +159,7 @@ query_result_from_kvtag(char *key_value_query, int8_t op_type) uint64_t *buf_ptr = NULL; input->op_type = op_type; input->attr_key = key_value_query; - assert(PDC_Server_dart_perform_one_server(input, output, &n_obj_ids, &buf_ptr) == SUCCEED); + PDC_Server_dart_perform_one_server(input, output, &n_obj_ids, &buf_ptr); LOG_JUST_PRINT("Query %s Successful! %d Results: ", key_value_query, n_obj_ids); for (int i = 0; i < n_obj_ids; i++) { LOG_JUST_PRINT("%llu, ", buf_ptr[i]); diff --git a/src/server/pdc_server_region/include/pdc_server_region_cache.h b/src/server/pdc_server_region/include/pdc_server_region_cache.h index 41f8836d6..a96a30466 100644 --- a/src/server/pdc_server_region/include/pdc_server_region_cache.h +++ b/src/server/pdc_server_region/include/pdc_server_region_cache.h @@ -19,6 +19,7 @@ #define PDC_MERGE_SUCCESS 5 pthread_mutex_t pdc_obj_cache_list_mutex; +extern int close_time_g; int PDC_region_server_cache_init(); int PDC_region_server_cache_finalize(); diff --git a/src/server/pdc_server_region/include/pdc_server_region_transfer.h b/src/server/pdc_server_region/include/pdc_server_region_transfer.h index 76e965671..b9c57d653 100644 --- a/src/server/pdc_server_region/include/pdc_server_region_transfer.h +++ b/src/server/pdc_server_region/include/pdc_server_region_transfer.h @@ -3,6 +3,30 @@ #include "pdc_region.h" +typedef enum pdc_region_writeout_strategy { + /** + * Store data as multiple regions inside a single file. + * Overlapping writes that are not fully contained append new data + * to the end of the file, with metadata tracking region locations. + * Supports incremental updates without rewriting large parts of the file. + */ + STORE_REGION_BY_REGION_SINGLE_FILE = 0, + + /** + * Store the entire object as a single flat file. + * Reads and writes operate by seeking directly within the file. + * No region metadata bookkeeping; simpler but less flexible for partial updates. + */ + STORE_FLATTENED_SINGLE_FILE, + + /** + * Store each flattened region in its own separate file. + * Enables independent file management per region. + * Useful for scenarios requiring isolated region access or storage. + */ + STORE_FLATTENED_REGION_PER_FILE +} pdc_region_writeout_strategy; + typedef struct transfer_request_all_data { uint64_t **obj_dims; uint64_t **remote_offset; @@ -13,6 +37,12 @@ typedef struct transfer_request_all_data { int * remote_ndim; char ** data_buf; int n_objs; + + pdc_var_type_t *var_types; + char ** json_filepaths; + char ** cur_state_str; + char ** client_state_str; + char ** store_state_str; } transfer_request_all_data; typedef struct pdc_transfer_request_status { @@ -34,9 +64,11 @@ perr_t PDC_server_transfer_request_init(); perr_t PDC_server_transfer_request_finalize(); -int try_reset_dims(); - -int get_server_rank(); +/** + * non-zero if the storage strategy supports resetting object dimensions + * zero otherwise + */ +int can_reset_dims(); /* * Create a new linked list node for a region transfer request and append it to the end of the linked list. diff --git a/src/server/pdc_server_region/pdc_server_data.c b/src/server/pdc_server_region/pdc_server_data.c index 53fdf1e84..97ae0410e 100644 --- a/src/server/pdc_server_region/pdc_server_data.c +++ b/src/server/pdc_server_region/pdc_server_data.c @@ -115,11 +115,17 @@ fill_storage_path(char *storage_location, pdcid_t obj_id) else stripe_count = pdc_nost_per_file_g; stripe_size = 16; // MB - PDC_Server_set_lustre_stripe(storage_location, stripe_count, stripe_size); - if (is_debug_g == 1 && pdc_server_rank_g == 0) { - LOG_DEBUG("storage_location is %s\n", storage_location); + static int have_print = 0; + if (!have_print) { + if (pdc_server_rank_g == 0) { + LOG_WARNING("Total Lustre OSTs, PDC Servers: %d, %d\n", lustre_total_ost_g, pdc_server_size_g); + LOG_WARNING("Setting Lustre stripe count to %d and size to %d MB\n", stripe_count, stripe_size); + } + have_print = 1; } + + PDC_Server_set_lustre_stripe(storage_location, stripe_count, stripe_size); #endif FUNC_LEAVE(0); @@ -4059,11 +4065,7 @@ PDC_Server_data_write_out(uint64_t obj_id, struct pdc_region_info *region_info, uint64_t i, j, pos; uint64_t * overlap_offset, *overlap_size; char * tmp_buf; -#if 0 - size_t total_write_size = 0, local_write_size; - int is_overlap; -#endif - FUNC_ENTER(NULL); + #ifdef PDC_TIMING double start = MPI_Wtime(), start_posix; #endif diff --git a/src/server/pdc_server_region/pdc_server_region_cache.c b/src/server/pdc_server_region/pdc_server_region_cache.c index 8344a1914..5d5119ba3 100644 --- a/src/server/pdc_server_region/pdc_server_region_cache.c +++ b/src/server/pdc_server_region/pdc_server_region_cache.c @@ -2,6 +2,10 @@ #include "pdc_server_region_cache.h" #include "pdc_timing.h" #include "pdc_logger.h" +#include "pdc_tf_profiler.h" +#include "pdc_tf_server.h" + +int close_time_g = 0; #ifdef PDC_SERVER_CACHE @@ -11,6 +15,12 @@ #define MAX_CACHE_SIZE_GB 32 #endif +#ifdef PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB +#define MAX_TRANSFORMATION_CACHE_SIZE_GB PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB +#else +#define MAX_TRANSFORMATION_CACHE_SIZE_GB 32 +#endif + #ifdef PDC_SERVER_IDLE_CACHE_FLUSH_TIME #define PDC_IDLE_CACHE_FLUSH_TIME_INT PDC_SERVER_IDLE_CACHE_FLUSH_TIME #else @@ -38,7 +48,9 @@ static pdc_obj_cache *obj_cache_list, *obj_cache_list_end; static pthread_t pdc_recycle_thread; static pthread_mutex_t pdc_cache_mutex; static int pdc_recycle_close_flag; +static size_t total_transformation_cache_size; static size_t total_cache_size; +static size_t maximum_transformation_cache_size; static size_t maximum_cache_size; static int pdc_idle_flush_time_g; @@ -53,7 +65,8 @@ PDC_region_server_cache_init() pdc_recycle_close_flag = 0; pthread_mutex_init(&pdc_obj_cache_list_mutex, NULL); pthread_mutex_init(&pdc_cache_mutex, NULL); - total_cache_size = 0; + total_cache_size = 0; + total_transformation_cache_size = 0; p = getenv("PDC_SERVER_CACHE_MAX_SIZE"); if (p != NULL) { @@ -63,6 +76,14 @@ PDC_region_server_cache_init() maximum_cache_size = MAX_CACHE_SIZE_GB * 1024llu * 1024llu * 1024llu; } + p = getenv("PDC_SERVER_TRANSFORMATION_CACHE_MAX_SIZE"); + if (p != NULL) { + maximum_transformation_cache_size = atol(p) * 1024llu * 1024llu * 1024llu; + } + else { + maximum_transformation_cache_size = maximum_cache_size; + } + p = getenv("PDC_SERVER_IDLE_CACHE_FLUSH_TIME"); if (p != NULL) pdc_idle_flush_time_g = atol(p); @@ -72,8 +93,10 @@ PDC_region_server_cache_init() #ifdef ENABLE_MPI MPI_Comm_rank(MPI_COMM_WORLD, &server_rank); #endif - if (server_rank == 0) + if (server_rank == 0) { LOG_INFO("Max cache size: %llu\n", maximum_cache_size); + LOG_INFO("Max transformation cache size: %llu\n", maximum_transformation_cache_size); + } obj_cache_list = NULL; obj_cache_list_end = NULL; @@ -98,7 +121,7 @@ PDC_region_server_cache_finalize() pdc_recycle_close_flag = 1; pthread_mutex_unlock(&pdc_cache_mutex); pthread_join(pdc_recycle_thread, NULL); - + close_time_g = 1; PDC_region_cache_flush_all(); pthread_mutex_destroy(&pdc_obj_cache_list_mutex); pthread_mutex_destroy(&pdc_cache_mutex); @@ -427,12 +450,11 @@ PDC_region_cache_copy(char *buf, char *buf2, const uint64_t *offset, const uint6 } /* - * This function cache metadata and data for a region write operation. + * This function caches metadata and data for a region write operation. * We store 1 object per element in the end of an array. Per object, there is a array of regions. The new * region is appended to the end of the region array after object searching by ID. This will result linear * search complexity for subregion search. */ - int PDC_region_cache_register(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, const char *buf, size_t buf_size, const uint64_t *offset, const uint64_t *size, int ndim, @@ -706,6 +728,34 @@ sort_by_offset(const void *elem1, const void *elem2) FUNC_LEAVE(0); #pragma GCC diagnostic pop } + +/** + * Returns a pointer to the region mapping for obj_id + * If no region mapping is found returns NULL + */ +static struct pdc_tf_obj_t * +PDCtf_get_region_mapping(pdcid_t obj_id, pdc_dg_t **dg) +{ + FUNC_ENTER(NULL); + + struct pdc_tf_obj_t * ret_value = NULL; + pdc_tf_obj_id_to_dg_t *obj_id_to_dg = NULL; + PDC_VECTOR_ITERATOR * tf_obj_id_to_dg_vector_iter = pdc_vector_iterator_new(tf_obj_id_to_dg_vector_g); + + while (pdc_vector_iterator_has_next(tf_obj_id_to_dg_vector_iter)) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = + (pdc_tf_obj_id_to_dg_t *)pdc_vector_iterator_next(tf_obj_id_to_dg_vector_iter); + if (cur_obj_id_to_dg->obj_id == obj_id) { + *dg = cur_obj_id_to_dg->dg; + PGOTO_DONE(&(cur_obj_id_to_dg->pdc_tf_obj)); + } + } + +done: + pdc_vector_iterator_destroy(tf_obj_id_to_dg_vector_iter); + FUNC_LEAVE(ret_value); +} + int PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache, int flag) { @@ -721,120 +771,111 @@ PDC_region_cache_flush_by_pointer(uint64_t obj_id, pdc_obj_cache *obj_cache, int uint64_t unit; struct pdc_region_info **obj_regions; char cur_time[64]; + int _pdc_server_rank_g = 0; + +#ifdef ENABLE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &_pdc_server_rank_g); +#endif + #ifdef PDC_TIMING double start_time = MPI_Wtime(); #endif + env_char = getenv("PDC_SERVER_CACHE_NO_FLUSH"); if (env_char && atoi(env_char) != 0) { - LOG_ERROR("Flushed disabled\n"); + if (_pdc_server_rank_g == 0) + LOG_ERROR("Flushed disabled\n"); FUNC_LEAVE(0); } - // For 1D case, we can merge regions to minimize the number of POSIX calls. - if (obj_cache->ndim == 1 && obj_cache->region_cache_size) { - start = (uint64_t *)PDC_malloc(sizeof(uint64_t) * obj_cache->region_cache_size * 2); - end = start + obj_cache->region_cache_size; - buf = (char **)PDC_malloc(sizeof(char *) * obj_cache->region_cache_size); - - // Sort the regions based on start index - obj_regions = (struct pdc_region_info **)PDC_malloc(sizeof(struct pdc_region_info *) * - obj_cache->region_cache_size); - unit = obj_cache->region_cache->region_cache_info->unit; - region_cache_iter = obj_cache->region_cache; - i = 0; - while (region_cache_iter) { - obj_regions[i] = region_cache_iter->region_cache_info; - region_cache_iter = region_cache_iter->next; - i++; - } - qsort(obj_regions, obj_cache->region_cache_size, sizeof(struct pdc_region_info *), sort_by_offset); - for (i = 0; i < obj_cache->region_cache_size; ++i) { - start[i] = obj_regions[i]->offset[0]; - end[i] = obj_regions[i]->offset[0] + obj_regions[i]->size[0]; - buf[i] = obj_regions[i]->buf; - } - obj_regions = (struct pdc_region_info **)PDC_free(obj_regions); - - // Merge adjacent regions - merge_requests(start, end, obj_cache->region_cache_size, buf, &new_start, &new_end, &new_buf, unit, - &merged_request_size); - start = (uint64_t *)PDC_free(start); - buf = (char **)PDC_free(buf); - // Record buffer pointer to be freed later. - buf_ptr = new_buf[0]; - // Override the first merge_request_size number of cache regions with the merge regions - obj_cache->region_cache_size = merged_request_size; - region_cache_iter = obj_cache->region_cache; - for (i = 0; i < merged_request_size; ++i) { - region_cache_info = region_cache_iter->region_cache_info; - region_cache_info->offset[0] = new_start[i]; - region_cache_info->size[0] = new_end[i] - new_start[i]; - region_cache_info->buf = (void *)PDC_free(region_cache_info->buf); - region_cache_info->buf = new_buf[i]; - if (i == merged_request_size - 1) { - region_cache_temp = region_cache_iter->next; - region_cache_iter->next = NULL; - region_cache_iter = region_cache_temp; - } - else { - region_cache_iter = region_cache_iter->next; - } - } - new_start = (uint64_t *)PDC_free(new_start); - new_buf = (char **)PDC_free(new_buf); - // Free other regions. - while (region_cache_iter) { - region_cache_info = region_cache_iter->region_cache_info; - region_cache_info->offset = (uint64_t *)PDC_free(region_cache_info->offset); - region_cache_info->buf = (void *)PDC_free(region_cache_info->buf); - region_cache_info = (struct pdc_region_info *)PDC_free(region_cache_info); - region_cache_temp = region_cache_iter; - region_cache_iter = region_cache_iter->next; - region_cache_temp = (pdc_region_cache *)PDC_free(region_cache_temp); - } - nflush += merged_request_size; - } // End for 1D + if (_pdc_server_rank_g == 0) + LOG_INFO("Starting flush for obj_id %" PRIu64 ", ndim=%d\n", obj_id, obj_cache->ndim); - // Iterate through all cache regions and use POSIX I/O to write them back to file system. region_cache_iter = obj_cache->region_cache; while (region_cache_iter != NULL) { + double region_start_time = MPI_Wtime(); + region_cache_info = region_cache_iter->region_cache_info; - PDC_Server_transfer_request_io(obj_id, obj_cache->ndim, obj_cache->dims, region_cache_info, - region_cache_info->buf, region_cache_info->unit, 1); - if (obj_cache->ndim >= 1) - write_size = region_cache_info->unit * region_cache_info->size[0]; - if (obj_cache->ndim >= 2) - write_size *= region_cache_info->size[1]; - if (obj_cache->ndim >= 3) - write_size *= region_cache_info->size[2]; - - if (write_size > 0) { + write_size = region_cache_info->unit; + for (int d = 0; d < obj_cache->ndim; d++) + write_size *= region_cache_info->size[d]; + + double mapping_start_time = MPI_Wtime(); + pdc_tf_region_mapping_t *region_mapping = NULL; + pdc_dg_t * dg = NULL; + struct pdc_tf_obj_t * tf_obj = PDCtf_get_region_mapping(obj_id, &dg); + double mapping_end_time = MPI_Wtime(); + + if (_pdc_server_rank_g == 0) + LOG_WARNING("Region mapping lookup took %.6f s\n", mapping_end_time - mapping_start_time); + + if (!PDCtf_region_has_attached_graph(tf_obj, obj_cache->ndim, unit, region_cache_info->offset, + region_cache_info->size, ®ion_mapping)) { + if (_pdc_server_rank_g == 0) + LOG_WARNING("No region mapping found for obj_id %" PRIu64 "\n", obj_id); + double io_start = MPI_Wtime(); + PDC_Server_transfer_request_io(obj_id, obj_cache->ndim, obj_cache->dims, region_cache_info, + region_cache_info->buf, region_cache_info->unit, 1); + double io_end = MPI_Wtime(); + if (_pdc_server_rank_g == 0) + LOG_WARNING("Flushed region via POSIX I/O in %.6f s\n", io_end - io_start); + } + else { + if (_pdc_server_rank_g == 0) + LOG_DEBUG("Found region mapping for obj_id %" PRIu64 "\n", obj_id); + + double io_start = MPI_Wtime(); + PDC_Server_transfer_request_io(obj_id, obj_cache->ndim, obj_cache->dims, region_cache_info, + region_cache_info->buf, region_cache_info->unit, 1); + double io_end = MPI_Wtime(); + if (_pdc_server_rank_g == 0) + LOG_WARNING("Flushed mapped region in %.6f s\n", io_end - io_start); + } + + if (write_size > 0 && _pdc_server_rank_g == 0) { PDC_get_time_str(cur_time); - LOG_INFO("Server flushed %.1f / %.1f MB to storage\n", write_size / 1048576.0, - total_cache_size / 1048576.0); + LOG_WARNING("[%s] Flushed %.2f / %.2f MB to storage\n", cur_time, write_size / 1048576.0, + total_cache_size / 1048576.0); } total_cache_size -= write_size; + + double free_start = MPI_Wtime(); region_cache_info->offset = (uint64_t *)PDC_free(region_cache_info->offset); if (obj_cache->ndim > 1) region_cache_info->buf = (void *)PDC_free(region_cache_info->buf); region_cache_info = (struct pdc_region_info *)PDC_free(region_cache_info); + region_cache_temp = region_cache_iter; region_cache_iter = region_cache_iter->next; region_cache_temp = (pdc_region_cache *)PDC_free(region_cache_temp); + double free_end = MPI_Wtime(); + + if (_pdc_server_rank_g == 0) + LOG_DEBUG("Memory cleanup for region took %.6f s\n", free_end - free_start); + nflush++; + + double region_end_time = MPI_Wtime(); + if (_pdc_server_rank_g == 0) + LOG_INFO("Region flush iteration took %.6f s\n", region_end_time - region_start_time); } + if (merged_request_size && obj_cache->ndim == 1) { buf_ptr = (char *)PDC_free(buf_ptr); } + obj_cache->region_cache = NULL; obj_cache->region_cache_size = 0; gettimeofday(&(obj_cache->timestamp), NULL); + #ifdef PDC_TIMING - pdc_server_timings->PDCcache_flush += MPI_Wtime() - start_time; + double total_flush_time = MPI_Wtime() - start_time; + pdc_server_timings->PDCcache_flush += total_flush_time; + if (server_rank == 0) + LOG_INFO("Total PDC_region_cache_flush_by_pointer time: %.6f s\n", total_flush_time); #endif - /* PDC_get_time_str(cur_time); */ FUNC_LEAVE(nflush); } @@ -937,15 +978,16 @@ PDC_region_cache_clock_cycle(void *ptr) obj_cache_iter = obj_cache_iter->next; pthread_mutex_unlock(&pdc_obj_cache_list_mutex); + // Gives the additional CPU time to other threads to avoid busy loop if there are many + // cache regions. usleep(300000); break; } obj_cache_iter = obj_cache_iter->next; - /* pthread_mutex_unlock(&pdc_obj_cache_list_mutex); */ + // Gives the CPU time to other threads to avoid busy loop if there are many cache regions. usleep(300000); } // End while obj_cache_iter - /* pthread_mutex_unlock(&pdc_obj_cache_list_mutex); */ - } // End if pdc_recycle_close_flag + } // End if pdc_recycle_close_flag else { pthread_mutex_unlock(&pdc_cache_mutex); break; @@ -990,51 +1032,69 @@ PDC_region_fetch(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, struct { FUNC_ENTER(NULL); - pdc_obj_cache *obj_cache = NULL, *obj_cache_iter; - int flag = 0; - // size_t j; + pdc_obj_cache * obj_cache = NULL, *obj_cache_iter; + int flag = 0; pdc_region_cache *region_cache_iter; uint64_t * overlap_offset, *overlap_size; + LOG_INFO("Fetching region for obj_id=%lu\n", obj_id); + + // Search for object in cache obj_cache_iter = obj_cache_list; while (obj_cache_iter != NULL) { if (obj_cache_iter->obj_id == obj_id) { obj_cache = obj_cache_iter; + LOG_INFO("Object %lu found in cache\n", obj_id); + break; // can break since we found it + } + else { + LOG_INFO("Checking against Object %lu\n", obj_cache_iter->obj_id); } obj_cache_iter = obj_cache_iter->next; } + if (obj_cache == NULL) { + LOG_INFO("Object %lu NOT found in cache\n", obj_id); + } + if (obj_cache != NULL) { - // Check if the input region is contained inside any cache region. + // Check if the input region is contained inside any cached region region_cache_iter = obj_cache->region_cache; while (region_cache_iter != NULL) { flag = detect_region_contained(region_info->offset, region_info->size, region_cache_iter->region_cache_info->offset, region_cache_iter->region_cache_info->size, region_info->ndim); if (flag) { - // flag = 1 means that the input region is fully contained in the cached region, so the return - // value of overlap_offset must not be NULL + LOG_INFO("Region fully contained in cache for obj_id=%lu\n", obj_id); + PDC_region_overlap_detect(region_info->ndim, region_info->offset, region_info->size, region_cache_iter->region_cache_info->offset, region_cache_iter->region_cache_info->size, &overlap_offset, &overlap_size); + memcpy_overlap_subregion(region_info->ndim, unit, region_cache_iter->region_cache_info->buf, region_cache_iter->region_cache_info->offset, region_cache_iter->region_cache_info->size, buf, region_info->offset, region_info->size, overlap_offset, overlap_size); + overlap_offset = (uint64_t *)PDC_free(overlap_offset); - // flag = 1 at here. break; } region_cache_iter = region_cache_iter->next; } + + if (!flag) { + LOG_INFO("Object %lu found but region NOT fully contained in cache\n", obj_id); + } } + if (!flag) { if (obj_cache != NULL) { + LOG_INFO("Flushing cached regions for obj_id=%lu before server fetch\n", obj_id); PDC_region_cache_flush_by_pointer(obj_id, obj_cache, 0); } + LOG_INFO("Fetching region from server for obj_id=%lu\n", obj_id); PDC_Server_transfer_request_io(obj_id, obj_ndim, obj_dims, region_info, buf, unit, 0); } - FUNC_LEAVE(0); } #endif diff --git a/src/server/pdc_server_region/pdc_server_region_request_handler.h b/src/server/pdc_server_region/pdc_server_region_request_handler.h index e16e921a5..887b99efe 100644 --- a/src/server/pdc_server_region/pdc_server_region_request_handler.h +++ b/src/server/pdc_server_region/pdc_server_region_request_handler.h @@ -2,6 +2,7 @@ #define PDC_SERVER_REGION_REQUEST_HANDLER_H #include "pdc_timing.h" +#include "pdc_tf_server.h" hg_return_t transfer_request_all_bulk_transfer_read_cb2(const struct hg_cb_info *info) @@ -61,7 +62,6 @@ transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info) handle_info = HG_Get_info(local_bulk_args->handle); request_data.n_objs = local_bulk_args->in.n_objs; parse_bulk_data(local_bulk_args->data_buf, &request_data, PDC_READ); - // print_bulk_data(&request_data); remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); total_mem_size = 0; @@ -92,8 +92,10 @@ transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info) remote_reg_info->size = request_data.remote_length[i]; mem_size = request_data.unit[i]; + LOG_DEBUG("Unit %d\n", request_data.unit[i]); for (j = 0; j < request_data.remote_ndim[i]; ++j) { mem_size *= request_data.remote_length[i][j]; + LOG_DEBUG("Remote length %d\n", remote_reg_info->offset[i]); } #ifdef PDC_SERVER_CACHE @@ -104,20 +106,12 @@ transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info) PDC_Server_transfer_request_io(request_data.obj_id[i], request_data.obj_ndim[i], request_data.obj_dims[i], remote_reg_info, (void *)ptr, request_data.unit[i], 0); -#endif -#if 0 - LOG_ERROR("server read array, offset = %lu, size = %lu:", request_data.remote_offset[i][0], request_data.remote_length[i][0]); uint64_t k; - for ( k = 0; k < remote_reg_info->size[0]; ++k ) { - LOG_ERROR("%d,", *(int*)(ptr + sizeof(int) * k)); - } - LOG_ERROR("\n"); #endif ptr += mem_size; } #ifndef PDC_SERVER_CACHE - for (i = 0; i < request_data.n_objs; ++i) { + for (i = 0; i < request_data.n_objs; ++i) PDC_Server_unregister_obj_region_by_pointer(temp_ptrs[i], 1); - } temp_ptrs = (data_server_region_t **)PDC_free(temp_ptrs); #endif @@ -138,7 +132,7 @@ transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info) ret = HG_Bulk_create(handle_info->hg_class, 1, &(local_bulk_args2->data_buf), &total_mem_size, HG_BULK_READWRITE, &(local_bulk_args2->bulk_handle)); if (ret != HG_SUCCESS) { - LOG_ERROR("Error at transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info)\n"); + LOG_ERROR("Error with HG_Bulk_create\n"); } // This is the actual data transfer. When transfer is finished, we are heading our way to the function @@ -147,9 +141,8 @@ transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info) HG_Bulk_transfer(handle_info->context, transfer_request_all_bulk_transfer_read_cb2, local_bulk_args2, HG_BULK_PUSH, handle_info->addr, local_bulk_args->in.local_bulk_handle, 0, local_bulk_args2->bulk_handle, 0, total_mem_size, HG_OP_ID_IGNORE); - if (ret != HG_SUCCESS) { - LOG_ERROR("Error at transfer_request_all_bulk_transfer_read_cb(const struct hg_cb_info *info)"); - } + if (ret != HG_SUCCESS) + LOG_ERROR("Error with HG_Bulk_transfer: %d\n", ret); // pointers in request_data are freed in the next call back function local_bulk_args->data_buf = (void *)PDC_free(local_bulk_args->data_buf); remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); @@ -188,7 +181,6 @@ transfer_request_all_bulk_transfer_write_cb(const struct hg_cb_info *info) remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); request_data.n_objs = local_bulk_args->in.n_objs; parse_bulk_data(local_bulk_args->data_buf, &request_data, PDC_WRITE); - // print_bulk_data(&request_data); #ifndef PDC_SERVER_CACHE data_server_region_t **temp_ptrs = @@ -212,15 +204,6 @@ transfer_request_all_bulk_transfer_write_cb(const struct hg_cb_info *info) request_data.obj_dims[i], remote_reg_info, (void *)request_data.data_buf[i], request_data.unit[i], 1); #endif - -#if 0 - uint64_t j; - LOG_ERROR("server write array, offset = %lu, size = %lu:", request_data.remote_offset[i][0], request_data.remote_length[i][0]); - for ( j = 0; j < remote_reg_info->size[0]; ++j ) { - LOG_ERROR("%d,", *(int*)(request_data.data_buf[i] + sizeof(int) * j)); - } - LOG_ERROR("\n"); -#endif pthread_mutex_lock(&transfer_request_status_mutex); PDC_finish_request(local_bulk_args->transfer_request_id[i]); pthread_mutex_unlock(&transfer_request_status_mutex); @@ -250,7 +233,6 @@ transfer_request_all_bulk_transfer_write_cb(const struct hg_cb_info *info) pdc_server_timings->PDCreg_transfer_request_inner_write_all_bulk_rpc += end - start; pdc_timestamp_register(pdc_transfer_request_inner_write_all_bulk_timestamps, start, end); #endif - FUNC_LEAVE(ret); } @@ -319,7 +301,7 @@ transfer_request_bulk_transfer_write_cb(const struct hg_cb_info *info) FUNC_ENTER(NULL); struct transfer_request_local_bulk_args *local_bulk_args = info->arg; - hg_return_t ret = HG_SUCCESS; + hg_return_t ret_value = HG_SUCCESS; struct pdc_region_info * remote_reg_info; uint64_t obj_dims[3]; @@ -345,24 +327,26 @@ transfer_request_bulk_transfer_write_cb(const struct hg_cb_info *info) remote_reg_info->ndim, remote_reg_info->ndim); PDC_copy_region_desc((local_bulk_args->in).obj_dims, obj_dims, remote_reg_info->ndim, remote_reg_info->ndim); -/* - printf("Server transfer request at write branch, index 1 value = %d\n", - *((int *)(local_bulk_args->data_buf + sizeof(int)))); -*/ + #ifdef PDC_SERVER_CACHE - PDC_transfer_request_data_write_out(local_bulk_args->in.obj_id, local_bulk_args->in.obj_ndim, obj_dims, - remote_reg_info, (void *)local_bulk_args->data_buf, - local_bulk_args->in.remote_unit); + if (PDC_transfer_request_data_write_out(local_bulk_args->in.obj_id, local_bulk_args->in.obj_ndim, + obj_dims, remote_reg_info, (void *)local_bulk_args->data_buf, + local_bulk_args->in.remote_unit) != SUCCEED) { + PGOTO_ERROR(HG_OTHER_ERROR, "Error with PDC_transfer_request_data_write_out"); + } #else - PDC_Server_transfer_request_io(local_bulk_args->in.obj_id, local_bulk_args->in.obj_ndim, obj_dims, - remote_reg_info, (void *)local_bulk_args->data_buf, - local_bulk_args->in.remote_unit, 1); + if (PDC_Server_transfer_request_io(local_bulk_args->in.obj_id, local_bulk_args->in.obj_ndim, obj_dims, + remote_reg_info, (void *)local_bulk_args->data_buf, + local_bulk_args->in.remote_unit, 1) != SUCCEED) { + PGOTO_ERROR(HG_OTHER_ERROR, "Error with PDC_Server_transfer_request_io"); + } #endif + +done: pthread_mutex_lock(&transfer_request_status_mutex); PDC_finish_request(local_bulk_args->transfer_request_id); pthread_mutex_unlock(&transfer_request_status_mutex); - local_bulk_args->data_buf = (void *)PDC_free(local_bulk_args->data_buf); - remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); + remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); HG_Bulk_free(local_bulk_args->bulk_handle); @@ -371,8 +355,7 @@ transfer_request_bulk_transfer_write_cb(const struct hg_cb_info *info) pdc_server_timings->PDCreg_transfer_request_inner_write_bulk_rpc += end - start; pdc_timestamp_register(pdc_transfer_request_inner_write_bulk_timestamps, start, end); #endif - - FUNC_LEAVE(ret); + FUNC_LEAVE(ret_value); } hg_return_t @@ -610,7 +593,6 @@ HG_TEST_RPC_CB(transfer_request_all, handle) pdc_timestamp_register(pdc_transfer_request_start_all_write_timestamps, start, end); } #endif - FUNC_LEAVE(ret_value); } @@ -732,9 +714,6 @@ HG_TEST_RPC_CB(transfer_request_metadata_query2, handle) FUNC_LEAVE(ret_value); } -/* static hg_return_t */ - -// transfer_request_cb(hg_handle_t handle) HG_TEST_RPC_CB(transfer_request, handle) { FUNC_ENTER(NULL); @@ -757,13 +736,10 @@ HG_TEST_RPC_CB(transfer_request, handle) HG_Get_input(handle, &in); info = HG_Get_info(handle); - total_mem_size = PDC_get_region_desc_size_bytes(in.remote_region.count, in.remote_unit, in.remote_region.ndim); - /* pthread_mutex_lock(&transfer_request_id_mutex); */ out.metadata_id = PDC_transfer_request_id_register(); - /* pthread_mutex_unlock(&transfer_request_id_mutex); */ pthread_mutex_lock(&transfer_request_status_mutex); PDC_commit_request(out.metadata_id); pthread_mutex_unlock(&transfer_request_status_mutex); @@ -781,22 +757,43 @@ HG_TEST_RPC_CB(transfer_request, handle) out.ret = 1; ret_value = HG_Respond(handle, NULL, NULL, &out); if (in.access_type == PDC_WRITE) { + if (in.pdc_tf_pkg.json_filepath != NULL && strlen(in.pdc_tf_pkg.json_filepath) > 0) { + LOG_DEBUG("RPC recieved region transfer with attached graph\n"); + + LOG_DEBUG("Region transfer json filepath: %s\n", in.pdc_tf_pkg.json_filepath); + LOG_DEBUG("Region transfer current state: %s\n", in.pdc_tf_pkg.cur_state); + LOG_DEBUG("Region transfer client state: %s\n", in.pdc_tf_pkg.client_state); + LOG_DEBUG("Region transfer stored state: %s\n", in.pdc_tf_pkg.store_state); + + if (PDCtf_store_json_mapping(in.obj_id, in.pdc_tf_pkg.json_filepath, in.pdc_tf_pkg.cur_state, + in.pdc_tf_pkg.client_state, in.pdc_tf_pkg.store_state, + in.remote_region.start, in.remote_region.count, + in.remote_region.ndim, in.pdc_tf_pkg.pdc_var_type) != SUCCEED) { + PGOTO_ERROR(HG_OTHER_ERROR, "Failed to PDCtf_store_json_mapping\n"); + } + } + else { + in.pdc_tf_pkg.json_filepath = NULL; + in.pdc_tf_pkg.cur_state = NULL; + in.pdc_tf_pkg.store_state = NULL; + } + ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); - if (ret_value != HG_SUCCESS) { - LOG_ERROR("Error at HG_TEST_RPC_CB(transfer_request, handle)\n"); - } + if (ret_value != HG_SUCCESS) + PGOTO_ERROR(ret_value, "Error with HG_Bulk_create"); // This is the actual data transfer. When transfer is finished, we are heading our way to the function // transfer_request_bulk_transfer_cb. ret_value = HG_Bulk_transfer(info->context, transfer_request_bulk_transfer_write_cb, local_bulk_args, HG_BULK_PULL, info->addr, in.local_bulk_handle, 0, local_bulk_args->bulk_handle, 0, total_mem_size, HG_OP_ID_IGNORE); + if (ret_value != HG_SUCCESS) + PGOTO_ERROR(ret_value, "Error with HG_Bulk_transfer"); } else { - remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); - + remote_reg_info = (struct pdc_region_info *)PDC_malloc(sizeof(struct pdc_region_info)); remote_reg_info->ndim = (in.remote_region).ndim; remote_reg_info->offset = (uint64_t *)PDC_malloc(remote_reg_info->ndim * sizeof(uint64_t)); remote_reg_info->size = (uint64_t *)PDC_malloc(remote_reg_info->ndim * sizeof(uint64_t)); @@ -809,30 +806,34 @@ HG_TEST_RPC_CB(transfer_request, handle) remote_reg_info->ndim); #ifdef PDC_SERVER_CACHE - PDC_transfer_request_data_read_from(in.obj_id, in.obj_ndim, obj_dims, remote_reg_info, - (void *)local_bulk_args->data_buf, in.remote_unit); + if (PDC_transfer_request_data_read_from(in.obj_id, in.obj_ndim, obj_dims, remote_reg_info, + (void *)local_bulk_args->data_buf, + in.remote_unit) != SUCCEED) { + PGOTO_ERROR(HG_OTHER_ERROR, "Error with PDC_transfer_request_data_read_from"); + } #else - PDC_Server_transfer_request_io(in.obj_id, in.obj_ndim, obj_dims, remote_reg_info, - (void *)local_bulk_args->data_buf, in.remote_unit, 0); + if (PDC_Server_transfer_request_io(in.obj_id, in.obj_ndim, obj_dims, remote_reg_info, + (void *)local_bulk_args->data_buf, in.remote_unit, 0) != SUCCEED) { + PGOTO_ERROR(HG_OTHER_ERROR, "Error with PDC_Server_transfer_request_io"); + } #endif ret_value = HG_Bulk_create(info->hg_class, 1, &(local_bulk_args->data_buf), (const hg_size_t *)&(local_bulk_args->total_mem_size), HG_BULK_READWRITE, &(local_bulk_args->bulk_handle)); - if (ret_value != HG_SUCCESS) { - LOG_ERROR("Error at HG_TEST_RPC_CB(transfer_request, handle)\n"); - } + if (ret_value != HG_SUCCESS) + PGOTO_ERROR(ret_value, "Error with HG_Bulk_create"); - // This is the actual data transfer. When transfer is finished, we are heading our way to the function - // transfer_request_bulk_transfer_cb. + // This is the actual data transfer ret_value = HG_Bulk_transfer(info->context, transfer_request_bulk_transfer_read_cb, local_bulk_args, HG_BULK_PUSH, info->addr, in.local_bulk_handle, 0, local_bulk_args->bulk_handle, 0, total_mem_size, HG_OP_ID_IGNORE); + if (ret_value != HG_SUCCESS) + PGOTO_ERROR(ret_value, "Error with HG_Bulk_transfer"); + remote_reg_info = (struct pdc_region_info *)PDC_free(remote_reg_info); } - if (ret_value != HG_SUCCESS) { - LOG_ERROR("Error at HG_TEST_RPC_CB(transfer_request, handle)\n"); - } +done: HG_Free_input(handle, &in); HG_Destroy(handle); @@ -847,7 +848,6 @@ HG_TEST_RPC_CB(transfer_request, handle) pdc_timestamp_register(pdc_transfer_request_start_write_timestamps, start, end); } #endif - FUNC_LEAVE(ret_value); } diff --git a/src/server/pdc_server_region/pdc_server_region_transfer.c b/src/server/pdc_server_region/pdc_server_region_transfer.c index 97950aaef..57fb43c57 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer.c @@ -1,15 +1,21 @@ +#include +#include + #include "pdc_client_server_common.h" #include "pdc_server_data.h" #include "pdc_timing.h" #include "pdc_logger.h" #include "pdc_malloc.h" +#include "pdc_tf_server.h" +#include "pdc_tf_common.h" +#include "pdc_vector.h" -static int io_by_region_g = 1; +static pdc_region_writeout_strategy storage_strategy_g = STORE_REGION_BY_REGION_SINGLE_FILE; int -try_reset_dims() +can_reset_dims() { - return io_by_region_g; + return storage_strategy_g == STORE_REGION_BY_REGION_SINGLE_FILE; } perr_t @@ -222,12 +228,11 @@ PDC_try_finish_request(uint64_t transfer_request_id, hg_handle_t handle, int *ha /* * Generate a remote transfer request ID in a very fast way. - * What happen if we have one request pending and call the register 2^64 times? This could result a repetitive - * transfer request ID generated. + * What happens if we have one request pending and call the register 2^64 times? This could result in a + * repetitive transfer request ID generated. * TODO: Scan the entire transfer list and search for repetitive nodes. * Not a thread-safe function, need protection from pthread_mutex_lock(&transfer_request_id_mutex); */ - pdcid_t PDC_transfer_request_id_register() { @@ -246,44 +251,27 @@ PDC_transfer_request_id_register() * Nonzero io_by_region_g will trigger region by region storage. Otherwise file flatten strategy is used */ #define PDC_POSIX_IO(fd, buf, io_size, is_write) \ + errno = 0; \ if (is_write) { \ if (write(fd, buf, io_size) != io_size) { \ - LOG_ERROR("server POSIX write failed\n"); \ + close(fd); \ + PGOTO_ERROR(FAIL, "Server POSIX write failed: %s", strerror(errno)); \ } \ } \ else { \ if (read(fd, buf, io_size) != io_size) { \ - LOG_ERROR("server POSIX read failed\n"); \ + close(fd); \ + PGOTO_ERROR(FAIL, "Server POSIX read failed: %s", strerror(errno)); \ } \ } -perr_t -PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, - struct pdc_region_info *region_info, void *buf, size_t unit, int is_write) +static inline char * +get_data_path() { FUNC_ENTER(NULL); - perr_t ret_value = SUCCEED; - int fd; - char * data_path = NULL; - char * user_specified_data_path = NULL; - char storage_location[ADDR_MAX]; - ssize_t io_size; - uint64_t i, j; - char cur_time[64]; - - if (io_by_region_g || obj_ndim == 0) { - // PDC_Server_register_obj_region(obj_id); - if (is_write) { - PDC_Server_data_write_out(obj_id, region_info, buf, unit); - } - else { - PDC_Server_data_read_from(obj_id, region_info, buf, unit); - } - PGOTO_DONE(ret_value); - } - if (obj_ndim != (int)region_info->ndim) - PGOTO_ERROR(FAIL, "Obj dim does not match obj dim\n"); + char *user_specified_data_path; + char *data_path; user_specified_data_path = getenv("PDC_DATA_LOC"); if (user_specified_data_path != NULL) { @@ -294,7 +282,29 @@ PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint64_t *ob if (data_path == NULL) data_path = "."; } - // Data path prefix will be $SCRATCH/pdc_data/$obj_id/ + + FUNC_LEAVE(data_path); +} + +static perr_t +PDC_Server_data_io_flattened(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, + struct pdc_region_info *region_info, void *buf, size_t unit, int is_write) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + int fd; + char * data_path = NULL; + char storage_location[ADDR_MAX]; + ssize_t io_size; + uint64_t i, j; + + if (obj_ndim != (int)region_info->ndim) + PGOTO_ERROR(FAIL, "Obj dim does not match region dim\n"); + + data_path = get_data_path(); + + // Data path prefix will be $SCRATCH/pdc_data/$obj_id/server$rank/s$rank.bin snprintf(storage_location, ADDR_MAX, "%.200s/pdc_data/%" PRIu64 "/server%d/s%04d.bin", data_path, obj_id, PDC_get_rank(), PDC_get_rank()); PDC_mkdir(storage_location); @@ -366,6 +376,519 @@ PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint64_t *ob FUNC_LEAVE(ret_value); } +/** + * Constructs file name for STORE_FLATTENED_REGION_PER_FILE storage strategy + * The returned pointer is heap allocated caller must free + */ +static char * +get_storage_location_region_per_file(int obj_id, int obj_ndim, const uint64_t *indices) +{ + FUNC_ENTER(NULL); + + char *storage_location = PDC_calloc(1, sizeof(char) * ADDR_MAX); + /** + * This is the filename suffix + * Each dimension can be a max of 20 character hence the DIM_MAX * 20 + * Also there is an '_' between each number and NULL terminator hence the DIM_MAX - 1 + 1 + */ + uint32_t storage_location_suffix_max_size = (obj_ndim * 20) + obj_ndim - 1 + 1; + // this is largest 64 bit number represented in base 10 assci and a NULL terminator hence + 1 + uint32_t uint64_t_max_assci_size = 20 + 1; + + // create temp strings and 0 out the data + char storage_location_suffix[storage_location_suffix_max_size]; + memset(storage_location_suffix, 0, storage_location_suffix_max_size); + char num_str[uint64_t_max_assci_size]; + memset(num_str, 0, uint64_t_max_assci_size); + + for (int i = 0; i < obj_ndim; i++) { + // NOTE: we validated earlier that file_dims[i] != 0 + snprintf(num_str, sizeof(num_str), "%" PRIu64, indices[i]); + strcat(storage_location_suffix, num_str); + // dont' add '_' unless there is another character after + if (i + 1 != obj_ndim) + strcat(storage_location_suffix, "_"); + } + + // Data path prefix will be $SCRATCH/pdc_data/$obj_id/server$rank/s$rank_$suffix.bin + snprintf(storage_location, ADDR_MAX, "%.200s/pdc_data/%" PRIu64 "/server%d/s%04d_%s.bin", get_data_path(), + obj_id, PDC_get_rank(), PDC_get_rank(), storage_location_suffix); + + PDC_mkdir(storage_location); + + FUNC_LEAVE(storage_location); +} + +#define FILE_START 0 +#define FILE_END 1 + +/* + * Converts a multi-dimensional index into a single linear index + * based on the provided dimensions. + * This function is used to map N-dimensional coordinates to a 1D index + * suitable for accessing flattened arrays + */ +static uint64_t +flatten_index(const uint64_t *indices, const uint64_t *dims, int ndim) +{ + FUNC_ENTER(NULL); + + uint64_t ret_value = 0; + uint64_t stride = 1; + for (int i = ndim - 1; i >= 0; i--) { + ret_value += indices[i] * stride; + stride *= dims[i]; + } + + FUNC_LEAVE(ret_value); +} + +static perr_t +PDC_Server_data_io_region_per_file(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, + const uint64_t *file_dims, struct pdc_region_info *region_info, void *buf, + size_t unit, int is_write) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + int dim, *fds = NULL; + char **temp_bufs = NULL, *all_temp_bufs = NULL, *temp_buf_ptr = NULL, *user_buf_ptr = NULL, + *storage_location = NULL; + uint64_t local_indices[obj_ndim], local_coords[obj_ndim], dims[obj_ndim], total_elements, + temp_bufs_array[obj_ndim][2], indices[obj_ndim], i, total_files, file_chunk_elements, + *elem_to_buf_idx = NULL, *elem_to_local_idx = NULL, file_indices_local[obj_ndim], buf_idx, local_idx, + e, remainder, coords[obj_ndim]; + size_t file_chunk_bytes; + ssize_t bytes_read, cur_bytes_read, bytes_written, cur_bytes_written; + + if (obj_ndim != (int)region_info->ndim) + PGOTO_ERROR(FAIL, "Obj dim does not match region dim"); + + for (i = 0; i < (uint64_t)obj_ndim; i++) { + if (file_dims[i] == 0) + PGOTO_ERROR(FAIL, "File dimension %d is zero", i); + } + + // Compute file index spans + for (i = 0; i < (uint64_t)obj_ndim; i++) { + temp_bufs_array[i][FILE_START] = region_info->offset[i] / file_dims[i]; + temp_bufs_array[i][FILE_END] = (region_info->offset[i] + region_info->size[i] - 1) / file_dims[i]; + } + + total_files = 1; + for (i = 0; i < (uint64_t)obj_ndim; i++) { + dims[i] = temp_bufs_array[i][FILE_END] - temp_bufs_array[i][FILE_START] + 1; + total_files *= dims[i]; + } + + file_chunk_elements = PDC_get_region_desc_size(file_dims, obj_ndim); + file_chunk_bytes = file_chunk_elements * unit; + + // Allocate contiguous temp buffer + all_temp_bufs = PDC_malloc(file_chunk_bytes); + temp_bufs = PDC_malloc(total_files * sizeof(char *)); + for (i = 0; i < total_files; i++) + temp_bufs[i] = all_temp_bufs + i * file_chunk_bytes; + fds = PDC_malloc(total_files * sizeof(int)); + + // Open files and read contents (pread avoids need for lseek) + for (i = 0; i < (uint64_t)obj_ndim; i++) + indices[i] = temp_bufs_array[i][FILE_START]; + + for (;;) { + storage_location = get_storage_location_region_per_file(obj_id, obj_ndim, indices); + + for (int d = 0; d < obj_ndim; d++) + local_indices[d] = indices[d] - temp_bufs_array[d][FILE_START]; + buf_idx = flatten_index(local_indices, dims, obj_ndim); + assert(buf_idx < total_files); + + errno = 0; + fds[buf_idx] = open(storage_location, O_RDWR | O_CREAT, 0644); + if (fds[buf_idx] < 0) { + if (errno == ENOENT) { + memset(temp_bufs[buf_idx], 0, file_chunk_bytes); + } + else { + storage_location = PDC_free(storage_location); + PGOTO_ERROR(FAIL, "Failed to open file %s: %s", storage_location, strerror(errno)); + } + } + else { + bytes_read = 0; + while ((size_t)bytes_read < file_chunk_bytes) { + cur_bytes_read = pread(fds[buf_idx], temp_bufs[buf_idx] + bytes_read, + file_chunk_bytes - bytes_read, bytes_read); + if (cur_bytes_read < 0) { + close(fds[buf_idx]); + PGOTO_ERROR(FAIL, "Failed to read file %s: %s", storage_location, strerror(errno)); + } + if (cur_bytes_read == 0) + break; // EOF + bytes_read += cur_bytes_read; + } + // zero-fill remainder if any + if ((size_t)bytes_read < file_chunk_bytes) + memset(temp_bufs[buf_idx] + bytes_read, 0, file_chunk_bytes - bytes_read); + } + + storage_location = PDC_free(storage_location); + + // Increment indices multi-dimensionally + dim = obj_ndim - 1; + while (dim >= 0) { + if (++indices[dim] <= temp_bufs_array[dim][FILE_END]) + break; + indices[dim] = temp_bufs_array[dim][FILE_START]; + dim--; + } + if (dim < 0) + break; + } + + // Precompute mappings for element-wise phase + total_elements = 1; + for (i = 0; i < (uint64_t)obj_ndim; i++) + total_elements *= region_info->size[i]; + + // Allocate maps + elem_to_buf_idx = PDC_malloc(total_elements * sizeof(uint64_t)); + elem_to_local_idx = PDC_malloc(total_elements * sizeof(uint64_t)); + + for (e = 0; e < total_elements; e++) { + remainder = e; + for (int d = obj_ndim - 1; d >= 0; d--) { + coords[d] = remainder % region_info->size[d]; + remainder /= region_info->size[d]; + coords[d] += region_info->offset[d]; + } + for (int d = 0; d < obj_ndim; d++) { + file_indices_local[d] = (coords[d] / file_dims[d]) - temp_bufs_array[d][FILE_START]; + local_coords[d] = coords[d] % file_dims[d]; + } + elem_to_buf_idx[e] = flatten_index(file_indices_local, dims, obj_ndim); + elem_to_local_idx[e] = flatten_index(local_coords, file_dims, obj_ndim); + } + + // Copy data + for (e = 0; e < total_elements; e++) { + buf_idx = elem_to_buf_idx[e]; + local_idx = elem_to_local_idx[e]; + temp_buf_ptr = temp_bufs[buf_idx] + local_idx * unit; + user_buf_ptr = (char *)buf + e * unit; + if (is_write) + memcpy(temp_buf_ptr, user_buf_ptr, unit); + else + memcpy(user_buf_ptr, temp_buf_ptr, unit); + } + + // Write back files if writing + if (is_write) { + // Reset indices + for (i = 0; i < (uint64_t)obj_ndim; i++) + indices[i] = temp_bufs_array[i][FILE_START]; + + for (;;) { + for (int d = 0; d < obj_ndim; d++) + local_indices[d] = indices[d] - temp_bufs_array[d][FILE_START]; + buf_idx = flatten_index(local_indices, dims, obj_ndim); + + // Write entire buffer at once + bytes_written = 0; + while ((size_t)bytes_written < file_chunk_bytes) { + cur_bytes_written = pwrite(fds[buf_idx], temp_bufs[buf_idx] + bytes_written, + file_chunk_bytes - bytes_written, bytes_written); + if (cur_bytes_written < 0) { + close(fds[buf_idx]); + PGOTO_ERROR(FAIL, "Failed to write file descriptor %d: %s", fds[buf_idx], + strerror(errno)); + } + bytes_written += cur_bytes_written; + } + close(fds[buf_idx]); + + // Increment indices multi-dimensionally + dim = obj_ndim - 1; + while (dim >= 0) { + if (++indices[dim] <= temp_bufs_array[dim][FILE_END]) + break; + indices[dim] = temp_bufs_array[dim][FILE_START]; + dim--; + } + if (dim < 0) + break; + } + } + +done: + if (temp_bufs) + temp_bufs = PDC_free(temp_bufs); + if (all_temp_bufs) + all_temp_bufs = PDC_free(all_temp_bufs); + if (fds) + fds = PDC_free(fds); + if (elem_to_buf_idx) + elem_to_buf_idx = PDC_free(elem_to_buf_idx); + if (elem_to_local_idx) + elem_to_local_idx = PDC_free(elem_to_local_idx); + + FUNC_LEAVE(ret_value); +} + +/** + * Returns a pointer to the region mapping for obj_id + * If no region mapping is found returns NULL + */ +struct pdc_tf_obj_t * +PDCtf_get_region_mapping(pdcid_t obj_id, pdc_dg_t **dg) +{ + FUNC_ENTER(NULL); + + struct pdc_tf_obj_t * ret_value = NULL; + pdc_tf_obj_id_to_dg_t *obj_id_to_dg = NULL; + PDC_VECTOR_ITERATOR * tf_obj_id_to_dg_vector_iter = pdc_vector_iterator_new(tf_obj_id_to_dg_vector_g); + + while (pdc_vector_iterator_has_next(tf_obj_id_to_dg_vector_iter)) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = + (pdc_tf_obj_id_to_dg_t *)pdc_vector_iterator_next(tf_obj_id_to_dg_vector_iter); + if (cur_obj_id_to_dg->obj_id == obj_id) { + *dg = cur_obj_id_to_dg->dg; + PGOTO_DONE(&(cur_obj_id_to_dg->pdc_tf_obj)); + } + } + +done: + pdc_vector_iterator_destroy(tf_obj_id_to_dg_vector_iter); + FUNC_LEAVE(ret_value); +} + +perr_t +PDC_Server_data_io_region_per_file_transformations(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, + struct pdc_region_info *region_info, void *buf, + size_t unit, int is_write, bool *ran_transformation) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + void * cpy_buf = NULL; + pdc_dg_t * dg = NULL; + struct pdc_tf_obj_t * tf_obj = NULL; + pdc_tf_region_mapping_t *region_mapping = NULL; + + // Check if we are already the desired + cpy_buf = buf; + + tf_obj = PDCtf_get_region_mapping(obj_id, &dg); + if (!PDCtf_region_has_attached_graph(tf_obj, region_info->ndim, unit, region_info->offset, + region_info->size, ®ion_mapping)) { + LOG_DEBUG("No region mapping found for obj_id %" PRIu64 "\n", obj_id); + *ran_transformation = false; + PGOTO_DONE(SUCCEED); + } + else { + LOG_DEBUG("Found region mapping for obj_id %" PRIu64 "\n", obj_id); + } + + assert(dg != NULL); + + pdc_tf_region_t output_region; + pdc_tf_region_t input_region; + if (is_write) + PDCtf_set_tf_region_t(&input_region, region_info->ndim, + region_mapping->conceptual_region.pdc_var_type, region_info->size); + else { + PDCtf_set_tf_region_t(&input_region, region_mapping->actual_region.ndim, + region_mapping->actual_region.pdc_var_type, region_mapping->actual_region.size); + } + + char *desired_state; + if (is_write) { + desired_state = region_mapping->region_state.store_state; + } + else { + desired_state = region_mapping->region_state.client_state; + + // Read in data for transformation + char *storage_location = get_storage_location_region_per_file(obj_id, obj_ndim, region_info->offset); + int fd = open(storage_location, O_RDONLY); + uint64_t bytes_to_read = PDC_get_region_desc_size_bytes( + input_region.size, PDC_get_var_type_size(input_region.pdc_var_type), input_region.ndim); + buf = PDC_malloc(bytes_to_read); + PDC_POSIX_IO(fd, buf, bytes_to_read, 0); + close(fd); + } + + // Get the flat conceptual offset of the region + uint64_t flat_conceptual_offset = + PDCtf_get_flat_conceptual_offset(obj_ndim, region_mapping->conceptual_offset, obj_dims); + + if (PDC_get_rank() == 0) + LOG_INFO("Executing graph with json filepath %s\n", dg->data); + // We can now execute the directed graph + if (strcmp(region_mapping->region_state.cur_state, desired_state) == 0) { + LOG_WARNING("Current state was equal to desired state\n"); + *ran_transformation = true; + PGOTO_DONE(SUCCEED); + } + else if (PDCtf_exec_graph(dg, flat_conceptual_offset, region_mapping->region_state.cur_state, + desired_state, input_region, &output_region, &buf, is_write, + PDC_TF_SCHED_STATIC) != SUCCEED) { + PGOTO_ERROR(FAIL, "Error with PDCtf_exec_graph"); + } + + // At this point we have run the transformation + *ran_transformation = true; + + if (is_write) { + // Write out data after transformation + char *storage_location = get_storage_location_region_per_file(obj_id, obj_ndim, region_info->offset); + int fd = open(storage_location, O_CREAT | O_WRONLY | O_TRUNC, 0644); + uint64_t bytes_to_write = PDC_get_region_desc_size_bytes( + output_region.size, PDC_get_var_type_size(output_region.pdc_var_type), output_region.ndim); + PDC_POSIX_IO(fd, buf, bytes_to_write, 1); + close(fd); + + // Update actual region mapping + PDCtf_copy_tf_region_t(&output_region, ®ion_mapping->actual_region); + } + else + memcpy(cpy_buf, buf, PDCtf_get_pdc_region_t_bytes(output_region)); + + // Updating the current state to the desired state on a write + if (is_write) { + region_mapping->region_state.cur_state = PDC_free(region_mapping->region_state.cur_state); + region_mapping->region_state.cur_state = strdup(desired_state); + } + +done: + FUNC_LEAVE(ret_value); +} + +static perr_t +PDC_shrink_file_dims(uint64_t *temp_file_dims, const uint64_t *obj_dims, uint8_t obj_ndim, size_t unit) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + for (int i = 0; i < obj_ndim; i++) { + temp_file_dims[i] = obj_dims[i]; + } + uint64_t max_bytes_per_file = 4096 * 4096 * 8; + + /** + * We need to reduce the region file size to a reasonable size + * The file size is malloced in this storage strategy possibly several times + * So we need to make sure this can fit in memory + * This following strategies halves the largest dimension until + * the file size is < max_bytes_per_file + */ + while (PDC_get_region_desc_size_bytes(temp_file_dims, unit, obj_ndim) > max_bytes_per_file) { + int max_dim = 0; + for (int i = 1; i < obj_ndim; i++) { + if (temp_file_dims[i] > temp_file_dims[max_dim]) + max_dim = i; + } + if (temp_file_dims[max_dim] <= 1) + PGOTO_ERROR(FAIL, "Cannot reduce dimension %d further", max_dim); + temp_file_dims[max_dim] /= 2; + } + +done: + FUNC_LEAVE(ret_value); +} + +/** + * This function is used to flush regions to the storage system + * If the cache is disabled it is called immediately on PDC_WRITE or PDC_READ + * If the cache is enabled it is called when evicting regions from the cache + * or when reading a region into the cache + */ +perr_t +PDC_Server_transfer_request_io(uint64_t obj_id, int obj_ndim, const uint64_t *obj_dims, + struct pdc_region_info *region_info, void *buf, size_t unit, int is_write) +{ + FUNC_ENTER(NULL); + + LOG_INFO("PDC_Server_transfer_request_io was called for obj_id: %lu\n", obj_id); + + perr_t ret_value = SUCCEED; + int my_rank = PDC_get_rank(); + + // --- Validate input parameters --- + if (obj_id == 0) + PGOTO_ERROR(FAIL, "obj_id is zero"); + if (!region_info) + PGOTO_ERROR(FAIL, "region_info is NULL"); + if (obj_ndim < 0 || obj_ndim > DIM_MAX) + PGOTO_ERROR(FAIL, "obj_ndim (%d) is invalid", obj_ndim); + if (!obj_dims && obj_ndim > 0) + PGOTO_ERROR(FAIL, "obj_dims is NULL but obj_ndim > 0"); + if (!buf) + PGOTO_ERROR(FAIL, "buf is NULL"); + if (unit == 0) + PGOTO_ERROR(FAIL, "unit is zero"); + + /** + * FIMXE: If running transformation need to validate that + * region info size and offset is flush with file_dims and + * has the same size as file_dimes. + * + * In addition, need a way for user's to set the file_dims + */ + // check if the obj has transformations + bool ran_transformation = false; + LOG_DEBUG("Checking for transformations for obj_id %" PRIu64 "\n", obj_id); + LOG_DEBUG("Region info offset %d\n", region_info->offset[0]); + if (PDC_Server_data_io_region_per_file_transformations(obj_id, obj_ndim, obj_dims, region_info, buf, unit, + is_write, &ran_transformation) != SUCCEED) { + PGOTO_ERROR(FAIL, "Error with PDC_Server_data_io_region_per_file_transformations"); + } + if (ran_transformation) { + if (my_rank == 0) + LOG_DEBUG("Ran %s storage strategy STORE_FLATTENED_REGION_PER_FILE_TRANSFORMATION\n", + (is_write) ? "write" : "read"); + PGOTO_DONE(SUCCEED); + } + + /** + * Switch between storage strategies and hand off to correct handler + */ + if (storage_strategy_g == STORE_REGION_BY_REGION_SINGLE_FILE || obj_ndim == 0) { + if (my_rank == 0) + LOG_DEBUG("Running %s storage strategy STORE_REGION_BY_REGION_SINGLE_FILE\n", + (is_write) ? "write" : "read"); + if (is_write) + PGOTO_DONE(PDC_Server_data_write_out(obj_id, region_info, buf, unit)); + else + PGOTO_DONE(PDC_Server_data_read_from(obj_id, region_info, buf, unit)); + } + else if (storage_strategy_g == STORE_FLATTENED_SINGLE_FILE) { + if (my_rank == 0) + LOG_DEBUG("Running %s storage strategy STORE_FLATTENED_SINGLE_FILE\n", + (is_write) ? "write" : "read"); + PGOTO_DONE( + PDC_Server_data_io_flattened(obj_id, obj_ndim, obj_dims, region_info, buf, unit, is_write)); + } + else if (storage_strategy_g == STORE_FLATTENED_REGION_PER_FILE) { + // FIXME: Need to find a reasonable size for this or hints from client + uint64_t temp_file_dims[DIM_MAX]; + if (PDC_shrink_file_dims(temp_file_dims, obj_dims, obj_ndim, unit) != SUCCEED) + PGOTO_ERROR(FAIL, "Error with PDC_shrink_file_dims"); + + if (my_rank == 0) + LOG_DEBUG("Running %s storage strategy STORE_FLATTENED_REGION_PER_FILE\n", + (is_write) ? "write" : "read"); + PGOTO_DONE(PDC_Server_data_io_region_per_file(obj_id, obj_ndim, obj_dims, temp_file_dims, region_info, + buf, unit, is_write)); + } + else + PGOTO_ERROR(FAIL, "Invalid storage strategy"); + +done: + FUNC_LEAVE(ret_value); +} + int clean_write_bulk_data(transfer_request_all_data *request_data) { @@ -399,6 +922,12 @@ parse_bulk_data(void *buf, transfer_request_all_data *request_data, pdc_access_t request_data->obj_dims = request_data->remote_length + request_data->n_objs; request_data->unit = (size_t *)PDC_malloc(sizeof(size_t) * request_data->n_objs); request_data->data_buf = (char **)PDC_malloc(sizeof(char *) * request_data->n_objs); + // preallocate transformations string arrays + request_data->var_types = (pdc_var_type_t *)PDC_calloc(1, sizeof(pdc_var_type_t) * request_data->n_objs); + request_data->json_filepaths = (char **)PDC_calloc(1, sizeof(char *) * request_data->n_objs); + request_data->client_state_str = (char **)PDC_calloc(1, sizeof(char *) * request_data->n_objs); + request_data->cur_state_str = (char **)PDC_calloc(1, sizeof(char *) * request_data->n_objs); + request_data->store_state_str = (char **)PDC_calloc(1, sizeof(char *) * request_data->n_objs); /* * The following times n_objs (one set per object). @@ -406,6 +935,7 @@ parse_bulk_data(void *buf, transfer_request_all_data *request_data, pdc_access_t * obj_ndim: sizeof(int) * remote remote_ndim: sizeof(int) * unit: sizeof(size_t) + * */ for (i = 0; i < request_data->n_objs; ++i) { request_data->obj_id[i] = *((pdcid_t *)ptr); @@ -414,8 +944,34 @@ parse_bulk_data(void *buf, transfer_request_all_data *request_data, pdc_access_t ptr += sizeof(int); request_data->remote_ndim[i] = *((int *)ptr); ptr += sizeof(int); - request_data->unit[i] = *((pdcid_t *)ptr); + request_data->unit[i] = *((size_t *)ptr); ptr += sizeof(size_t); + request_data->var_types[i] = *((pdc_var_type_t *)ptr); + ptr += sizeof(pdc_var_type_t); + + LOG_DEBUG("obj_id: %d, ndim: %d, remote_ndim: %d, unit: %zu, var_type: %d\n", request_data->obj_id[i], + request_data->obj_ndim[i], request_data->remote_ndim[i], request_data->unit[i], + request_data->var_types[i]); + + // Parse and print strings immediately after unit + if (access_type == PDC_WRITE) { + request_data->json_filepaths[i] = ptr; + if (strlen(ptr) > 0) + LOG_DEBUG("Object %d json_filepath: %s\n", i, request_data->json_filepaths[i]); + ptr += strlen(ptr) + 1; + request_data->cur_state_str[i] = ptr; + if (strlen(ptr) > 0) + LOG_DEBUG("Object %d cur_state: %s\n", i, request_data->cur_state_str[i]); + ptr += strlen(ptr) + 1; + request_data->client_state_str[i] = ptr; + if (strlen(ptr) > 0) + LOG_DEBUG("Object %d client_state: %s\n", i, request_data->client_state_str[i]); + ptr += strlen(ptr) + 1; + request_data->store_state_str[i] = ptr; + if (strlen(ptr) > 0) + LOG_DEBUG("Object %d store_state: %s\n", i, request_data->store_state_str[i]); + ptr += strlen(ptr) + 1; + } } /* * For each of objects @@ -431,6 +987,29 @@ parse_bulk_data(void *buf, transfer_request_all_data *request_data, pdc_access_t ptr += request_data->remote_ndim[i] * sizeof(uint64_t); request_data->obj_dims[i] = (uint64_t *)ptr; ptr += request_data->obj_ndim[i] * sizeof(uint64_t); + + // Setup transformations + if (request_data->json_filepaths[i] != NULL && strlen(request_data->json_filepaths[i]) > 0) { + LOG_DEBUG("RPC recieved region transfer with attached graph\n"); + + LOG_DEBUG("Parse region transfer json filepath: %s\n", request_data->json_filepaths[i]); + LOG_DEBUG("Parse region transfer current state: %s\n", request_data->cur_state_str[i]); + LOG_DEBUG("Parse region transfer client state: %s\n", request_data->client_state_str[i]); + LOG_DEBUG("Parse region transfer stored state: %s\n", request_data->store_state_str[i]); + + LOG_DEBUG("Object %d: remote_offset[0]=%lu, remote_length[0]=%lu, ndim=%d, obj_dims[0]=%lu\n", + request_data->obj_id[i], request_data->remote_offset[i][0], + request_data->remote_length[i][0], request_data->obj_ndim[i], + request_data->obj_dims[i][0]); + if (PDCtf_store_json_mapping(request_data->obj_id[i], request_data->json_filepaths[i], + request_data->cur_state_str[i], request_data->client_state_str[i], + request_data->store_state_str[i], request_data->remote_offset[i], + request_data->remote_length[i], request_data->remote_ndim[i], + request_data->var_types[i]) != SUCCEED) { + LOG_ERROR("Failed to PDCtf_store_json_mapping\n"); + } + } + if (access_type == PDC_WRITE) { data_size = request_data->remote_length[i][0] * request_data->unit[i]; for (j = 1; j < request_data->remote_ndim[i]; ++j) { diff --git a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c index c66ecd905..d8924d15c 100644 --- a/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c +++ b/src/server/pdc_server_region/pdc_server_region_transfer_metadata_query.c @@ -57,6 +57,123 @@ static uint64_t metadata_query_buf_create(pdc_obj_region_metadata *regions, int * Entry function for this class. Should be only called once at the beginning of Server init. * If checkpoint is not NULL, then load previously checkpointed metadata to static variables. */ +/*perr_t +transfer_request_metadata_query_init(int pdc_server_size_input, char *checkpoint) +{ + FUNC_ENTER(NULL); + + LOG_INFO("Entered transfer_request_metadata_query_init\n"); + + hg_return_t ret_value = HG_SUCCESS; + char *ptr; + int n_objs, reg_count; + int i, j; + + metadata_server_objs = NULL; + metadata_server_objs_end = NULL; + metadata_query_buf_head = NULL; + metadata_query_buf_end = NULL; + pdc_server_size = pdc_server_size_input; + LOG_INFO("pdc_server_size set to %d\n", pdc_server_size); + + data_server_bytes = (uint64_t *)PDC_calloc(pdc_server_size, sizeof(uint64_t)); + LOG_INFO("Allocated data_server_bytes\n"); + + query_id_g = 100000; + LOG_INFO("query_id_g initialized to %d\n", query_id_g); + + ptr = checkpoint; + pthread_mutex_init(&metadata_query_mutex, NULL); + LOG_INFO("metadata_query_mutex initialized\n"); + + if (checkpoint) { + LOG_INFO("Checkpoint provided\n"); + + n_objs = *(int *)ptr; + ptr += sizeof(int); + LOG_INFO("Number of objects in checkpoint: %d\n", n_objs); + + for (i = 0; i < n_objs; ++i) { + LOG_INFO("Processing object %d\n", i); + + // Allocate first object node + if (metadata_server_objs) { + metadata_server_objs_end->next = + (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + metadata_server_objs_end = metadata_server_objs_end->next; + LOG_INFO("Allocated new metadata_server_objs node\n"); + } + else { + metadata_server_objs = (pdc_obj_metadata_pkg *)PDC_malloc(sizeof(pdc_obj_metadata_pkg)); + metadata_server_objs_end = metadata_server_objs; + LOG_INFO("Allocated first metadata_server_objs node\n"); + } + + metadata_server_objs_end->next = NULL; + metadata_server_objs_end->obj_id = *(uint64_t *)ptr; + ptr += sizeof(uint64_t); + LOG_INFO("Object ID: %lu\n", metadata_server_objs_end->obj_id); + + metadata_server_objs_end->ndim = *(int *)ptr; + ptr += sizeof(int); + LOG_INFO("Object ndim: %d\n", metadata_server_objs_end->ndim); + + reg_count = *(int *)ptr; + ptr += sizeof(int); + LOG_INFO("Region count: %d\n", reg_count); + + // Allocate first region + metadata_server_objs_end->regions = + (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); + metadata_server_objs_end->regions_end = metadata_server_objs_end->regions; + metadata_server_objs_end->regions_end->next = NULL; + + // Copy first region data + metadata_server_objs_end->regions_end->reg_offset = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * metadata_server_objs_end->ndim * 2); + metadata_server_objs_end->regions_end->reg_size = + metadata_server_objs_end->regions_end->reg_offset + metadata_server_objs_end->ndim; + + metadata_server_objs_end->regions_end->data_server_id = *(uint32_t *)ptr; + ptr += sizeof(uint32_t); + memcpy(metadata_server_objs_end->regions_end->reg_offset, ptr, + sizeof(uint64_t) * metadata_server_objs_end->ndim * 2); + ptr += sizeof(uint64_t) * metadata_server_objs_end->ndim * 2; + + LOG_INFO("Region 0 data_server_id: %u\n", metadata_server_objs_end->regions_end->data_server_id); + LOG_INFO("Region 0 reg_offset copied\n"); + + // Allocate remaining regions + for (j = 1; j < reg_count; ++j) { + LOG_INFO("Processing region %d for object %d\n", j, i); + + metadata_server_objs_end->regions_end->next = + (pdc_region_metadata_pkg *)PDC_malloc(sizeof(pdc_region_metadata_pkg)); + metadata_server_objs_end->regions_end = metadata_server_objs_end->regions_end->next; + metadata_server_objs_end->regions_end->next = NULL; + + metadata_server_objs_end->regions_end->reg_offset = + (uint64_t *)PDC_malloc(sizeof(uint64_t) * metadata_server_objs_end->ndim * 2); + metadata_server_objs_end->regions_end->reg_size = + metadata_server_objs_end->regions_end->reg_offset + metadata_server_objs_end->ndim; + + metadata_server_objs_end->regions_end->data_server_id = *(uint32_t *)ptr; + ptr += sizeof(uint32_t); + memcpy(metadata_server_objs_end->regions_end->reg_offset, ptr, + sizeof(uint64_t) * metadata_server_objs_end->ndim * 2); + ptr += sizeof(uint64_t) * metadata_server_objs_end->ndim * 2; + + LOG_INFO("Region %d data_server_id: %u\n", j, + metadata_server_objs_end->regions_end->data_server_id); + LOG_INFO("Region %d reg_offset copied\n", j); + } + } + } + + LOG_INFO("Exiting transfer_request_metadata_query_init\n"); + FUNC_LEAVE(ret_value); +}*/ + perr_t transfer_request_metadata_query_init(int pdc_server_size_input, char *checkpoint) { diff --git a/src/server/transform/include/pdc_tf_builtin_common.h b/src/server/transform/include/pdc_tf_builtin_common.h new file mode 100644 index 000000000..97321b5dc --- /dev/null +++ b/src/server/transform/include/pdc_tf_builtin_common.h @@ -0,0 +1,66 @@ +#ifndef PDC_TF_BUILTIN_COMMON_H +#define PDC_TF_BUILTIN_COMMON_H + +#include +#include + +// FIXME: this should be picked up from the cmakelists +#define ENABLE_TF_ZFP_COMPRESSION +// FIMXE: same +#define ENABLE_TF_SECRET_BOX_ENCRYPTION +// FIXME: same +#define ENABLE_TF_SZ_COMPRESSION +// FIXME: same +#define ENABLE_TF_SZ_GPU_COMPRESSSION +// FIXME: same +#define ENABLE_TF_TURBO_COMPRESSION + +#include "pdc_tf_common.h" + +#ifdef ENABLE_TF_SZ_GPU_COMPRESSSION +bool pdc_tf_builtin_sz_compress_cuda(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +bool pdc_tf_builtin_sz_decompress_cuda(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +#endif +#ifdef ENABLE_TF_SZ_COMPRESSION +bool pdc_tf_builtin_sz_compress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); +bool pdc_tf_builtin_sz_decompress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); +#endif +#ifdef ENABLE_TF_ZFP_COMPRESSION +bool pdc_tf_builtin_zfp_compress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); +bool pdc_tf_builtin_zfp_decompress(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +#endif +#if defined(ENABLE_TF_ZFP_COMPRESSION) && defined(CUDA_ENABLED) +bool pdc_tf_builtin_zfp_compress_cuda(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +bool pdc_tf_builtin_zfp_decompress_cuda(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +#endif + +#ifdef ENABLE_TF_SECRET_BOX_ENCRYPTION +bool pdc_tf_builtin_encrypt(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); +bool pdc_tf_builtin_decrypt(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); +#endif + +#ifdef ENABLE_TF_TURBO_COMPRESSION +bool pdc_tf_builtin_turbo_compress(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +bool pdc_tf_builtin_turbo_decompress(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region); +#endif + +#endif diff --git a/src/server/transform/include/pdc_tf_common.h b/src/server/transform/include/pdc_tf_common.h new file mode 100644 index 000000000..f2ad8ed23 --- /dev/null +++ b/src/server/transform/include/pdc_tf_common.h @@ -0,0 +1,237 @@ +#ifndef PDC_TF_COMMON_H +#define PDC_TF_COMMON_H + +#include "mercury_proc_string.h" + +#include "pdc_public.h" +#include "pdc_dg.h" +#include "pdc_region.h" +#include "pdc_obj_pkg.h" +#include "pdc_vector.h" +#include "pdc_tf_user.h" + +typedef struct pdc_tf_region_state_t { + pdcid_t dg_id; + char * cur_state; + char * client_state; + char * store_state; +} pdc_tf_region_state_t; + +typedef struct pdc_tf_region_mapping_t { + pdc_tf_region_state_t region_state; + pdc_tf_region_t conceptual_region; + uint64_t conceptual_offset[DIM_MAX]; + + // This is the region information for storing on disk + pdc_tf_region_t actual_region; +} pdc_tf_region_mapping_t; + +/** + * This is a field in _pdc_obj_info that enables transformations + * This is a mapping between conceptual and actual regions + * Example: Conceptual region is the region before compression, + * actual region is the region after compression. + * The conceptual region is used to define the transformation, + * while the actual region is used to store the data. + * The num_region is the number of regions with attached graphs. + * The attach_to_all_regions indicates whether all region transfers + * should go through the directed graph. + */ +typedef struct pdc_tf_obj_t { + /** + * These fields are for attaching graphs to region transfers + * after the graph has been attached to the entire object + */ + bool attach_to_all_regions; + pdc_tf_region_state_t all_regions_state; + + // This field is used to attach graphs to individual regions + PDC_VECTOR *region_mappings_vector; +} pdc_obj_tf_t; + +extern char *pdc_tf_dev_strs[]; +extern char *pdc_tf_location_strs[]; + +/** + * Strings needed by server to run transformation + */ +typedef struct pdc_tf_pkg_t { + uint32_t pdc_var_type; + hg_string_t json_filepath; + hg_string_t cur_state; + hg_string_t client_state; + hg_string_t store_state; +} pdc_tf_pkg_t; + +void append_host_to_dev_time(pdc_tf_builtin_func_t *func, double value); +void append_dev_to_host_time(pdc_tf_builtin_func_t *func, double value); +void append_exec_time(pdc_tf_builtin_func_t *func, double value); + +double get_host_to_dev_avg(const pdc_tf_builtin_func_t *func); +double get_dev_to_host_avg(const pdc_tf_builtin_func_t *func); +double get_exec_avg(const pdc_tf_builtin_func_t *func); + +// this is our global array of builtin functions +extern PDC_VECTOR *pdc_tf_builtin_funcs_vector_g; + +/** + * @brief Initializes a pdc_tf_region_t with the given dimensionality, type, and sizes. + * + * @param dest Pointer to the region struct to populate. + * @param ndim Number of dimensions. + * @param pdc_var_type Element type (e.g. PDC_FLOAT, PDC_DOUBLE). + * @param size Array of per-dimension sizes (must have at least ndim elements). + * @return SUCCEED on success, FAIL otherwise. + */ +perr_t PDCtf_set_tf_region_t(pdc_tf_region_t *dest, uint8_t ndim, pdc_var_type_t pdc_var_type, + uint64_t *size); + +/** + * @brief Deep-copies a pdc_tf_region_t from src into dest. + * + * Copies ndim, pdc_var_type, and all size values. + * + * @param src Source region to copy from. + * @param dest Destination region to copy into. + * @return SUCCEED on success, FAIL otherwise. + */ +perr_t PDCtf_copy_tf_region_t(pdc_tf_region_t *src, pdc_tf_region_t *dest); + +/** + * @brief Parses a JSON file and constructs a transformation directed graph (DG). + * + * The JSON file must describe a set of named states and functions connecting them, + * with each function specifying its device (CPU/GPU), location (builtin/external), + * input state, output state, and optional parameters. External functions are loaded + * via dlopen/dlsym using the lib_path field. + * + * @param filepath Path to the JSON file describing the transformation graph. + * @return Pointer to the newly created pdc_dg_t, or NULL on failure. + */ +pdc_dg_t *PDCtf_dg_json_create_common(char *filepath); + +/** + * @brief Registers all compiled-in builtin transform functions into the global function registry. + * + * Which functions are registered depends on compile-time feature flags: + * - ENABLE_TF_SZ_GPU_COMPRESSSION → SZ GPU compress/decompress + * - ENABLE_TF_SZ_COMPRESSION → SZ CPU compress/decompress + * - ENABLE_TF_ZFP_COMPRESSION → ZFP CPU compress/decompress + * - CUDA_ENABLED → ZFP GPU compress/decompress + * - ENABLE_TF_SECRET_BOX_ENCRYPTION→ SecretBox encrypt/decrypt + * - ENABLE_TF_TURBO_COMPRESSION → Turbo compress/decompress + * + * Must be called before any transform graph is used. + * + * @return SUCCEED on success, FAIL otherwise. + */ +perr_t PDCtf_init_builtin_funcs(); + +/** + * @brief Registers a single function pointer in the global builtin function registry. + * + * Associates a name and device type with a C function pointer so it can later + * be looked up by PDCtf_link_builtin_func(). + * + * @param func_name Name to register the function under (e.g. "zfp_compress"). + * @param c_func The function pointer to register. + * @param dev Device the function runs on (PDC_TF_CPU_DEVICE or PDC_TF_GPU_DEVICE). + * @return SUCCEED on success, FAIL otherwise. + */ +perr_t PDCtf_add_builtin_func(char *func_name, c_func_t c_func, pdc_tf_dev_t dev); + +/** + * @brief Resolves a named builtin function and binds it to a pdc_tf_func_t. + * + * Searches the global builtin registry for a function matching both the given + * name and device, and sets f->c_func to the matching function pointer. + * + * @param func_name Name of the function to look up (e.g. "sz_decompress"). + * @param dev Device to match against (PDC_TF_CPU_DEVICE or PDC_TF_GPU_DEVICE). + * @param f Output pdc_tf_func_t whose c_func field will be set on success. + * @return SUCCEED if found and linked, FAIL otherwise. + */ +perr_t PDCtf_link_builtin_func(char *func_name, pdc_tf_dev_t dev, pdc_tf_func_t *f); + +/** + * @brief Checks whether a region of a PDC transform object has an attached transformation graph. + * + * Iterates over all region mappings on the given transform object and checks whether + * any conceptual region matches the provided ndim, unit, offset, and size. If a match + * is found, region_mapping is set to point to it. + * + * @param tf_obj The transform object to search. + * @param ndim Number of dimensions of the query region. + * @param unit Element size in bytes (used to match pdc_var_type). + * @param offset Per-dimension offsets of the query region. + * @param size Per-dimension sizes of the query region. + * @param region_mapping Output pointer set to the matching region mapping if found. + * @return true if a matching region mapping was found, false otherwise. + */ +bool PDCtf_region_has_attached_graph(struct pdc_tf_obj_t *tf_obj, int ndim, size_t unit, uint64_t *offset, + uint64_t *size, pdc_tf_region_mapping_t **region_mapping); + +/** + * @brief Returns the total number of elements in a region (product of all dimension sizes). + * + * @param reg The region descriptor. + * @return Total element count across all dimensions. + */ +size_t PDCtf_get_pdc_region_t_elements(pdc_tf_region_t reg); + +/** + * @brief Computes a flat (linear) offset from a multi-dimensional offset array. + * + * Uses row-major (C-order) stride calculation over the provided dimension sizes. + * + * @param ndim Number of dimensions (must be > 0). + * @param offset Per-dimension offset values. + * @param dims Per-dimension sizes used to compute strides. + * @return Flat linear offset corresponding to the multi-dimensional offset. + */ +size_t PDCtf_get_flat_conceptual_offset(int ndim, uint64_t offset[4], const uint64_t *dims); + +/** + * @brief Returns the total size in bytes of all elements in a region. + * + * Equivalent to PDCtf_get_pdc_region_t_elements(reg) * PDC_get_var_type_size(reg.pdc_var_type). + * + * @param reg The region descriptor. + * @return Total byte size of the region's data. + */ +size_t PDCtf_get_pdc_region_t_bytes(pdc_tf_region_t reg); + +/** + * @brief Logs the contents of a pdc_tf_region_t for debugging purposes. + * + * Currently a no-op stub; intended to print ndim, type, and size array. + * + * @param reg The region to log. + */ +void PDCtf_log_pdc_region_t(pdc_tf_region_t reg); + +/** + * @brief Prints the sequence of transform functions along the shortest path between two states. + * + * Runs Dijkstra/BFS on the directed graph from cur_state to desired_state and + * logs each edge (function) along the resulting path. + * + * @param dg The transformation directed graph to search. + * @param cur_state Name of the starting state. + * @param desired_state Name of the target state. + */ +void PDCtf_print_exec_path_common(pdc_dg_t *dg, char *cur_state, char *desired_state); + +/** + * @brief Prints the full transformation graph in Graphviz DOT format. + * + * Outputs all edges with their function names and device (CPU=blue, GPU=red). + * If write_to_file is true, output is redirected to "graph.txt"; otherwise + * it goes to stdout. + * + * @param dg The transformation directed graph to print. + * @param write_to_file If true, write DOT output to "graph.txt" instead of stdout. + */ +void PDCtf_print_dg_common(pdc_dg_t *dg, bool write_to_file); + +#endif // PDC_TF_COMMON_H diff --git a/src/server/transform/include/pdc_tf_poly_sched.h b/src/server/transform/include/pdc_tf_poly_sched.h new file mode 100644 index 000000000..99d695da6 --- /dev/null +++ b/src/server/transform/include/pdc_tf_poly_sched.h @@ -0,0 +1,101 @@ +/* + * pdc_tf_poly_sched.h + * + * Polynomial-based GPU selection scheduler for PDC Data Flyway. + * + * Uses a degree-3 polynomial fitted offline on 2,400 data points of ZFP GPU + * compression under varying GEMM contention levels to predict total + * transformation time (H2D + compression + D2H) for each available GPU. + * The GPU with the lowest predicted total_ms is selected. + * + * Model features (6 total): + * x0 = data_size_mb -- size of data to transform + * x1 = gpu_util -- GPU utilization % (0-100) + * x2 = power_mw -- GPU power draw in milliwatts + * x3 = mem_used_mb -- GPU memory used in MB + * x4 = prev_d2h_ms -- previous D2H transfer time in ms + * x5 = prev_total_ms -- previous total transformation time in ms + * + * R² = 0.9853 MAE = 2.209 ms (Rank 15 from exhaustive feature search) + * + * Polynomial evaluation: + * The degree-3 polynomial over 6 features produces terms of the form + * x_i^a * x_j^b * x_k^c where a+b+c <= 3. The coefficients are loaded + * from a file generated by export_coefficients.py at initialization time. + * + * Usage: + * 1. Call pdc_tf_poly_sched_init() at server startup. + * 2. Call pdc_tf_poly_select_gpu() before each transformation to get the + * best GPU index. + * 3. Call pdc_tf_poly_update() after each transformation to record lag. + * 4. Call pdc_tf_poly_sched_finalize() at server shutdown. + */ + +#ifndef PDC_TF_POLY_SCHED_H +#define PDC_TF_POLY_SCHED_H + +#include +#include "pdc_tf_profiler.h" + +/* ── polynomial model constants ──────────────────────────────────────────── */ +#define PDC_POLY_N_FEATURES 4 /* number of input features */ +#define PDC_POLY_DEGREE 3 /* polynomial degree */ + +/* Maximum number of polynomial terms for degree=3, n_features=6: + * C(6+3, 3) = C(9,3) = 84 terms */ +#define PDC_POLY_MAX_TERMS 84 + +/* ── polynomial model state ──────────────────────────────────────────────── */ +typedef struct { + double coefficients[PDC_POLY_MAX_TERMS]; /* fitted coefficients */ + int n_terms; /* actual number of terms */ + int powers[PDC_POLY_MAX_TERMS][PDC_POLY_N_FEATURES]; /* exponent matrix */ + int initialized; /* 1 if loaded successfully */ +} pdc_tf_poly_model_t; + +extern pdc_tf_poly_model_t pdc_tf_poly_model; + +/* ── init / finalize ───────────────────────────────────────────────────── + * Load polynomial coefficients from coeff_file (generated by + * export_coefficients.py). Must be called before pdc_tf_poly_select_gpu(). + * + * Returns 0 on success, -1 on failure. + * ─────────────────────────────────────────────────────────────────────── */ +int pdc_tf_poly_sched_init(const char *coeff_file); +void pdc_tf_poly_sched_finalize(void); + +/* ── predict total transformation time for one GPU ─────────────────────── + * Given a feature vector, evaluate the degree-3 polynomial and return + * the predicted total_ms. Returns -1.0 on error. + * + * features[0] = data_size_mb + * features[1] = gpu_util (0-100) + * features[2] = power_mw + * features[3] = mem_used_mb + * features[4] = prev_d2h_ms (-1.0 if no prior observation) + * features[5] = prev_total_ms (-1.0 if no prior observation) + * ─────────────────────────────────────────────────────────────────────── */ +double pdc_tf_poly_predict(const double features[PDC_POLY_N_FEATURES]); + +/* ── select best GPU ───────────────────────────────────────────────────── + * Queries NVML for each GPU, builds the feature vector, evaluates the + * polynomial, and returns the index of the GPU with lowest predicted + * total_ms. + * + * data_size_mb : size of the region to be transformed in MB + * + * Returns GPU device index (0 to nvml_device_count-1). + * Falls back to lowest-utilization GPU if model is not initialized. + * ─────────────────────────────────────────────────────────────────────── */ +int pdc_tf_poly_select_gpu(double data_size_mb); + +/* ── update lag features after transformation ──────────────────────────── + * Record observed timings for device_index so they become lag features + * for the next scheduling decision on that device. + * + * Wraps pdc_tf_update_device_lag() from pdc_tf_profiler.h. + * ─────────────────────────────────────────────────────────────────────── */ +void pdc_tf_poly_update(unsigned int device_index, double h2d_ms, double comp_ms, double d2h_ms, + double total_ms); + +#endif /* PDC_TF_POLY_SCHED_H */ \ No newline at end of file diff --git a/src/server/transform/include/pdc_tf_profiler.h b/src/server/transform/include/pdc_tf_profiler.h new file mode 100644 index 000000000..067a0ded3 --- /dev/null +++ b/src/server/transform/include/pdc_tf_profiler.h @@ -0,0 +1,57 @@ +#ifndef PDC_TF_PROFILER_H +#define PDC_TF_PROFILER_H + +#include +#include "pdc_logger.h" +#include "pdc_timing.h" +#include "pdc_malloc.h" + +#define PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE 1 /* single fresh sample */ + +extern int pdc_tf_profiler_init; +extern int pdc_tf_profiler_nvml_init; +extern int pdc_tf_profiler_cpu_init; +extern unsigned int pdc_tf_profiler_nvml_device_count; + +typedef struct { + double gpu_utilization; + int memory_utilization; + unsigned long memory_total; + unsigned long memory_used; + unsigned long memory_free; + unsigned int power_mw; +} pdc_tf_profiler_nvml_sample_t; + +typedef struct { + double cpu_utilization; +} pdc_tf_profiler_cpu_sample_t; + +typedef struct { + pdc_tf_profiler_nvml_sample_t *nvml_samples[PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE]; + int nvml_head; + pdc_tf_profiler_cpu_sample_t * cpu_samples[PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE]; + int cpu_head; + unsigned int nvml_device_count; +} pdc_tf_profiler_samples_t; + +extern pdc_tf_profiler_samples_t pdc_tf_profiler_samples; + +/* main update — called from server loop */ +perr_t pdc_tf_update_profiler(double elapsed_total_time_sec, double elapsed_progress_time_sec); + +/* fresh NVML sample — called immediately before scheduling decision */ +perr_t pdc_tf_nvml_profiler_update(void); + +/* utilization queries */ +double pdc_tf_avg_gpu_utilization(unsigned int device_index); +double pdc_tf_avg_cpu_utilization(void); +double pdc_tf_avg_gpu_power_mw(unsigned int device_index); +unsigned long pdc_tf_avg_gpu_mem_used(unsigned int device_index); + +/* lag storage for polynomial scheduler */ +void pdc_tf_update_device_lag(unsigned int device_index, double h2d_ms, double comp_ms, double d2h_ms, + double total_ms); +void pdc_tf_get_device_lag(unsigned int device_index, double *prev_h2d_ms, double *prev_comp_ms, + double *prev_d2h_ms, double *prev_total_ms); + +#endif /* PDC_TF_PROFILER_H */ \ No newline at end of file diff --git a/src/server/transform/include/pdc_tf_server.h b/src/server/transform/include/pdc_tf_server.h new file mode 100644 index 000000000..2fc9e052b --- /dev/null +++ b/src/server/transform/include/pdc_tf_server.h @@ -0,0 +1,63 @@ +#ifndef PDC_TF_H +#define PDC_TF_H + +#include "pdc_tf_common.h" +#include "pdc_logger.h" +#include "pdc_timing.h" +#include "pdc_vector.h" + +extern PDC_VECTOR *tf_obj_id_to_dg_vector_g; + +/** + * This is similar to the client side pdc_tf_obj_t + * The mean difference being that in the former + * the structure is a field on _pdc_obj_info which has the + * object id. We need to keep track of it on the server side. + * Hence the extra obj_id field here. + */ +typedef struct pdc_tf_obj_id_to_dg_t { + pdcid_t obj_id; + struct pdc_tf_obj_t pdc_tf_obj; + pdc_dg_t * dg; +} pdc_tf_obj_id_to_dg_t; + +/** + * These functions should only be used on the + * data server. There are similar functions between + * this and the client API, however, the method in + * which directed graphs and their relation to regions + * is stored does not use the pdcid_t on the server side. + */ + +/** + * Load the JSON filepath into a directed graph and + * creates a mapping for the conceptual to actual region representation + * NOTE: This does not set the actual reigon size because that should be + * set after the transformations have been run when writing to the data server + * and before transformations have been run when reading from the data server + */ +perr_t PDCtf_store_json_mapping(pdcid_t obj_id, char *json_filepath, char *cur_state, char *client_state, + char *store_state, uint64_t *offset, uint64_t *size, uint8_t ndim, + pdc_var_type_t pdc_var_type); + +/* ── scheduling mode enum ──────────────────────────────────────────────────── + * Pass to PDCtf_exec_graph() to select scheduling strategy. + * + * PDC_TF_SCHED_DYNAMIC : polynomial-based GPU selection (default) + * - Predicts total_ms for each GPU using the fitted degree-3 polynomial + * - Selects the GPU with lowest predicted total_ms + * - Updates lag features after each transformation + * + * PDC_TF_SCHED_STATIC : always use GPU 0 (baseline for comparison) + * - Mirrors the behavior of USE_GPU=1 with a fixed device + * - Used to measure scheduler benefit vs no scheduling + * ────────────────────────────────────────────────────────────────────────── */ +typedef enum { + PDC_TF_SCHED_DYNAMIC = 0, + PDC_TF_SCHED_STATIC = 1, +} pdc_tf_sched_mode_t; + +perr_t PDCtf_exec_graph(pdc_dg_t *dg, uint64_t flat_conceptual_offset, char *cur_state, char *desired_state, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region, void **input, + int is_write, pdc_tf_sched_mode_t sched_mode); +#endif diff --git a/src/server/transform/include/pdc_tf_user.h b/src/server/transform/include/pdc_tf_user.h new file mode 100644 index 000000000..26e4c3929 --- /dev/null +++ b/src/server/transform/include/pdc_tf_user.h @@ -0,0 +1,223 @@ +/** + * NOTICE: This file is included in a custom transformation libaries. + * It should not include any headers that are not available to the custom transformation library. + * It should also not include any headers that define symbols that + * may conflict with symbols in the custom transformation library. + */ +#ifndef PDC_TF_USER_H +#define PDC_TF_USER_H + +#define NUM_TF_FUNC_TIMES 5 + +#include "pdc_vector.h" +#include "pdc_dg.h" + +// Specifies what device the function can run on +typedef enum pdc_tf_dev_t { PDC_TF_CPU_DEVICE, PDC_TF_GPU_DEVICE, PDC_TF_NUM_DEVICES } pdc_tf_dev_t; + +typedef struct pdc_tf_state_t { + char * name; + PDC_VECTOR *pdc_tf_dg_params_vector; +} pdc_tf_state_t; + +// Specifies whether a function is internal or external. +typedef enum pdc_tf_location_t { PDC_TF_BUILTIN, PDC_TF_EXTERNAL, PDC_TF_NUM_LOCATIONS } pdc_tf_location_t; +extern char *pdc_tf_location_strs[]; + +#define DIM_MAX 4 + +typedef struct pdc_tf_region_t { + size_t ndim; + uint8_t pdc_var_type; + uint64_t size[DIM_MAX]; +} pdc_tf_region_t; + +typedef struct pdc_tf_internal_param { + pdc_dg_t *dg; + uint64_t flat_conceptual_offset; + double host_to_dev_time; + double dev_to_host_time; +} pdc_tf_internal_param; + +/** + * Prototype for region transformation functions + * + * Before the function is invoked, `output_state.tf_region` is set to `input_state.tf_region`, so if the + * transformation does not change the region size, the user does not need to + * modify `output_state.tf_region`. + * + * `region_data` is a double pointer to the input region's data buffer. + * The function may either mutate the existing buffer in place or allocate a new + * buffer and update `*region_data` to point to it. + * + * If a new data buffer is assigned to `*region_data`, it must be heap-allocated + * so that PDC can free it. The original pointer should NOT be freed. + */ +typedef bool (*c_func_t)(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region); + +typedef struct pdc_tf_builtin_func_t { + char * name; + pdc_tf_dev_t dev; + pdc_tf_location_t location; + c_func_t c_func; + char * params_str; + PDC_VECTOR * pdc_tf_dg_params_vector; + uint32_t cur_host_to_dev_avg_time_index; + uint32_t cur_dev_to_host_avg_time_index; + uint32_t cur_exec_avg_time_index; + double host_to_dev_avg_time[NUM_TF_FUNC_TIMES]; + double dev_to_host_avg_time[NUM_TF_FUNC_TIMES]; + double exec_avg_time[NUM_TF_FUNC_TIMES]; +} pdc_tf_builtin_func_t; + +typedef pdc_tf_builtin_func_t pdc_tf_func_t; +/** + * Used to store parameters for states and edges + * within the directed graph. + * + * The conceptual ID (which is the flat offset) + * is used to identify the parameters + * for a specific region. + */ +typedef struct pdc_tf_dg_params_t { + uint64_t flat_conceptual_offset; + void * params; + uint64_t params_size; +} pdc_tf_dg_params_t; + +/** + * Note: true is success and false is failure for the functions in this file. + * This is because the custom transformation library. + */ +bool PDCtf_set_state_param(pdc_dg_t *dg, char *state_name, uint64_t flat_conceptual_offset, void *params, + uint64_t params_size); +bool PDCtf_get_state_param(pdc_dg_t *dg, char *state_name, uint64_t flat_conceptual_offset, void **params, + uint64_t *params_size); +bool PDCtf_set_func_param(pdc_dg_t *dg, char *func_name, pdc_tf_dev_t dev, uint64_t flat_conceptual_offset, + void *params, uint64_t params_size); +bool PDCtf_get_func_param(pdc_dg_t *dg, char *func_name, pdc_tf_dev_t dev, uint64_t flat_conceptual_offset, + void **params, uint64_t *params_size); +/** + * @brief Set state parameters. + * + * This macro wraps the call to PDCtf_set_state_param to set the value + * of a named state parameter. + * + * @param name [in] The name of the state parameter to set. + * @param params [in] Pointer to the data to set for the parameter. + * @param params_size [in] Size in bytes of the data pointed to by params. + */ +#define SET_STATE_PARAMS(name, params, params_size) \ + do { \ + PDCtf_set_state_param(internal_param.dg, name, internal_param.flat_conceptual_offset, params, \ + params_size); \ + } while (0) + +/** + * @brief Get state parameters. + * + * @param name The name of the state parameter to retrieve. + * @param params [in,out] Pointer to a buffer pointer that will be set by the function. + * @param params_size [in,out] Pointer to a size_t that will be set to the size of the returned buffer. + */ +#define GET_STATE_PARAMS(name, params, params_size) \ + do { \ + PDCtf_get_state_param(internal_param.dg, name, internal_param.flat_conceptual_offset, params, \ + params_size); \ + } while (0) + +/** + * @brief Set func parameters. + * + * This macro wraps the call to PDCtf_set_func_param to set the value + * of a named func parameter. + * + * @param name [in] The name of the func parameter to set. + * @param params [in] Pointer to the data to set for the parameter. + * @param params_size [in] Size in bytes of the data pointed to by params. + */ +#define SET_FUNC_PARAMS(name, dev, params, params_size) \ + do { \ + PDCtf_set_func_param(internal_param->dg, name, dev, internal_param->flat_conceptual_offset, params, \ + params_size); \ + } while (0) + +/** + * @brief Get func parameters. + * + * @param name The name of the func parameter to retrieve. + * @param params [in,out] Pointer to a buffer pointer that will be set by the function. + * @param params_size [in,out] Pointer to a size_t that will be set to the size of the returned buffer. + */ +#define GET_FUNC_PARAMS(name, dev, params, params_size) \ + do { \ + PDCtf_get_func_param(internal_param->dg, name, dev, internal_param->flat_conceptual_offset, params, \ + params_size); \ + } while (0) + +/** + * @brief Start timing a host-to-device transfer. + * + * This macro records the start timestamp for a host-to-device transfer + * into a stack-allocated timespec. Pair with END_HOST_TO_DEV_TIME to + * accumulate the elapsed duration into internal_param.host_to_dev_time. + */ +#define START_HOST_TO_DEV_TIME() \ + struct timespec __host_to_dev_start_time__; \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &__host_to_dev_start_time__); \ + } while (0) + +/** + * @brief End timing a host-to-device transfer. + * + * This macro records the end timestamp and accumulates the elapsed time + * since the matching START_HOST_TO_DEV_TIME() call into + * internal_param.host_to_dev_time. + * + * @note May be called multiple times if there are multiple transfer + * segments; each call adds to the running total. + */ +#define END_HOST_TO_DEV_TIME() \ + do { \ + struct timespec __host_to_dev_end_time__; \ + clock_gettime(CLOCK_MONOTONIC, &__host_to_dev_end_time__); \ + internal_param->host_to_dev_time += \ + (__host_to_dev_end_time__.tv_sec - __host_to_dev_start_time__.tv_sec) + \ + (__host_to_dev_end_time__.tv_nsec - __host_to_dev_start_time__.tv_nsec) / 1e9; \ + } while (0) + +/** + * @brief Start timing a device-to-host transfer. + * + * This macro records the start timestamp for a device-to-host transfer + * into a stack-allocated timespec. Pair with END_DEV_TO_HOST_TIME to + * accumulate the elapsed duration into internal_param.dev_to_host_time. + */ +#define START_DEV_TO_HOST_TIME() \ + struct timespec __dev_to_host_start_time__; \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &__dev_to_host_start_time__); \ + } while (0) + +/** + * @brief End timing a device-to-host transfer. + * + * This macro records the end timestamp and accumulates the elapsed time + * since the matching START_DEV_TO_HOST_TIME() call into + * internal_param.dev_to_host_time. + * + * @note May be called multiple times if there are multiple transfer + * segments; each call adds to the running total. + */ +#define END_DEV_TO_HOST_TIME() \ + do { \ + struct timespec __dev_to_host_end_time__; \ + clock_gettime(CLOCK_MONOTONIC, &__dev_to_host_end_time__); \ + internal_param->dev_to_host_time += \ + (__dev_to_host_end_time__.tv_sec - __dev_to_host_start_time__.tv_sec) + \ + (__dev_to_host_end_time__.tv_nsec - __dev_to_host_start_time__.tv_nsec) / 1e9; \ + } while (0) + +#endif \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_encrypt_cpu.c b/src/server/transform/pdc_tf_builtin_encrypt_cpu.c new file mode 100644 index 000000000..f3fde0b54 --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_encrypt_cpu.c @@ -0,0 +1,95 @@ +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_SECRET_BOX_ENCRYPTION +#include + +unsigned char key[crypto_secretbox_KEYBYTES] = {0}; +unsigned char nonce[crypto_secretbox_NONCEBYTES] = {0}; + +typedef struct encrypt_params_t { + pdc_tf_region_t unencrypted_region; +} encrypt_params_t; + +bool +pdc_tf_builtin_encrypt(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_encrypt called\n"); + PDCtf_log_pdc_region_t(input_region); + + size_t plaintext_len = PDCtf_get_pdc_region_t_bytes(input_region); + size_t ciphertext_len = plaintext_len + crypto_secretbox_MACBYTES; + unsigned char *ciphertext = malloc(ciphertext_len); + if (!ciphertext) { + LOG_ERROR("Failed to allocate ciphertext buffer\n"); + return false; + } + + if (crypto_secretbox_easy(ciphertext, (unsigned char *)*region_data, plaintext_len, nonce, key) != 0) { + LOG_ERROR("Encryption failed\n"); + free(ciphertext); + return false; + } + + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = ciphertext_len; + + encrypt_params_t *out_params = (encrypt_params_t *)malloc(sizeof(encrypt_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->unencrypted_region); + SET_FUNC_PARAMS("secret_box_encrypt", PDC_TF_CPU_DEVICE, out_params, sizeof(encrypt_params_t)); + + *region_data = ciphertext; + LOG_DEBUG("Encryption succeeded, ciphertext length: %zu bytes\n", ciphertext_len); + return true; +} + +bool +pdc_tf_builtin_decrypt(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_decrypt called\n"); + PDCtf_log_pdc_region_t(input_region); + + size_t ciphertext_len = PDCtf_get_pdc_region_t_bytes(input_region); + + encrypt_params_t *in_params; + uint64_t in_params_size; + GET_FUNC_PARAMS("secret_box_encrypt", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + + if (ciphertext_len < crypto_secretbox_MACBYTES) { + LOG_ERROR("Ciphertext too short\n"); + return false; + } + + size_t plaintext_len = PDC_get_region_desc_size_bytes( + in_params->unencrypted_region.size, PDC_get_var_type_size(in_params->unencrypted_region.pdc_var_type), + in_params->unencrypted_region.ndim); + + unsigned char *plaintext = malloc(plaintext_len); + if (!plaintext) { + LOG_ERROR("Failed to allocate plaintext buffer\n"); + return false; + } + + if (crypto_secretbox_open_easy(plaintext, (unsigned char *)*region_data, ciphertext_len, nonce, key) != + 0) { + LOG_ERROR("Decryption failed or ciphertext tampered\n"); + free(plaintext); + return false; + } + + PDCtf_copy_tf_region_t(&in_params->unencrypted_region, output_region); + *region_data = plaintext; + + LOG_DEBUG("Decryption succeeded, plaintext length: %zu bytes\n", plaintext_len); + return true; +} + +#endif // ENABLE_TF_SECRET_BOX_ENCRYPTION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_sz_cpu.c b/src/server/transform/pdc_tf_builtin_sz_cpu.c new file mode 100644 index 000000000..498199f6e --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_sz_cpu.c @@ -0,0 +1,127 @@ +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_SZ_COMPRESSION +#include "sz3c.h" + +typedef struct sz_compress_params_t { + pdc_tf_region_t decompressed_region; +} sz_compress_params_t; + +bool +pdc_tf_builtin_sz_compress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_sz_compress was called\n"); + PDCtf_log_pdc_region_t(input_region); + + int sz_type; + switch (input_region.pdc_var_type) { + case PDC_FLOAT: + sz_type = SZ_FLOAT; + break; + case PDC_DOUBLE: + sz_type = SZ_DOUBLE; + break; + case PDC_INT: + case PDC_INT32: + sz_type = SZ_FLOAT; + break; + case PDC_INT64: + sz_type = SZ_DOUBLE; + break; + default: + LOG_ERROR("Unsupported element type for SZ3: %d\n", input_region.pdc_var_type); + return false; + } + + int err_bound_mode = ABS; + double absErrBound = 0.01; + double relBoundRatio = 0.0; + double pwrBoundRatio = 0.0; + + size_t r1 = input_region.size[0]; + size_t r2 = (input_region.ndim > 1) ? input_region.size[1] : 1; + size_t r3 = (input_region.ndim > 2) ? input_region.size[2] : 1; + size_t r4 = (input_region.ndim > 3) ? input_region.size[3] : 1; + size_t r5 = (input_region.ndim > 4) ? input_region.size[4] : 1; + + size_t compressed_size = 0; + unsigned char *compressed = + SZ_compress_args(sz_type, *region_data, &compressed_size, err_bound_mode, absErrBound, relBoundRatio, + pwrBoundRatio, r5, r4, r3, r2, r1); + if (!compressed) { + LOG_ERROR("SZ3 compression failed\n"); + return false; + } + + *region_data = compressed; + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = compressed_size; + + sz_compress_params_t *out_params = (sz_compress_params_t *)PDC_malloc(sizeof(sz_compress_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("sz_compress", PDC_TF_CPU_DEVICE, out_params, sizeof(sz_compress_params_t)); + + LOG_INFO("SZ3 compression succeeded, compressed size: %zu bytes\n", compressed_size); + return true; +} + +bool +pdc_tf_builtin_sz_decompress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_sz_decompress was called\n"); + PDCtf_log_pdc_region_t(input_region); + + sz_compress_params_t *in_params = NULL; + uint64_t in_params_size; + GET_FUNC_PARAMS("sz_compress", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + PDCtf_log_pdc_region_t(in_params->decompressed_region); + + int sz_type; + switch (in_params->decompressed_region.pdc_var_type) { + case PDC_FLOAT: + sz_type = SZ_FLOAT; + break; + case PDC_DOUBLE: + sz_type = SZ_DOUBLE; + break; + case PDC_INT: + case PDC_INT32: + sz_type = SZ_FLOAT; + break; + case PDC_INT64: + sz_type = SZ_DOUBLE; + break; + default: + LOG_ERROR("Unsupported element type for SZ3: %d\n", in_params->decompressed_region.pdc_var_type); + return false; + } + + size_t r1 = in_params->decompressed_region.size[0]; + size_t r2 = (in_params->decompressed_region.ndim > 1) ? in_params->decompressed_region.size[1] : 1; + size_t r3 = (in_params->decompressed_region.ndim > 2) ? in_params->decompressed_region.size[2] : 1; + size_t r4 = (in_params->decompressed_region.ndim > 3) ? in_params->decompressed_region.size[3] : 1; + size_t r5 = (in_params->decompressed_region.ndim > 4) ? in_params->decompressed_region.size[4] : 1; + + void *decompressed = SZ_decompress(sz_type, *region_data, input_region.size[0], r5, r4, r3, r2, r1); + if (!decompressed) { + LOG_ERROR("SZ3 decompression failed\n"); + return false; + } + + *region_data = decompressed; + PDCtf_copy_tf_region_t(&in_params->decompressed_region, output_region); + + LOG_INFO("SZ3 decompression succeeded\n"); + return true; +} + +#endif // ENABLE_TF_SZ_COMPRESSION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_sz_gpu.c b/src/server/transform/pdc_tf_builtin_sz_gpu.c new file mode 100644 index 000000000..e6857b168 --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_sz_gpu.c @@ -0,0 +1,160 @@ +#include +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_SZ_GPU_COMPRESSSION +#include "cusz.h" +#include + +typedef struct sz_compress_params_t { + pdc_tf_region_t decompressed_region; +} sz_compress_params_t; + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err = call; \ + if (err != cudaSuccess) { \ + fprintf(stderr, "[CUDA ERROR] %s:%d: %s (code %d)\n", __FILE__, __LINE__, \ + cudaGetErrorString(err), err); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +bool +pdc_tf_builtin_sz_compress_cuda(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_sz_compress_cuda called\n"); + PDCtf_log_pdc_region_t(input_region); + + psz_dtype dtype; + switch (input_region.pdc_var_type) { + case PDC_FLOAT: + dtype = F4; + break; + case PDC_DOUBLE: + dtype = F8; + break; + case PDC_INT32: + dtype = F4; + break; + case PDC_INT64: + dtype = F8; + break; + default: + LOG_ERROR("Unsupported element type for SZ: %d\n", input_region.pdc_var_type); + return false; + } + + uint64_t decompressed_bytes = PDC_get_region_desc_size_bytes( + input_region.size, PDC_get_var_type_size(input_region.pdc_var_type), input_region.ndim); + + float *d_data = NULL; + cudaMalloc((void **)&d_data, decompressed_bytes); + cudaMemcpy(d_data, *region_data, decompressed_bytes, cudaMemcpyHostToDevice); + + psz_len3 len = {1, 1, 1}; + if (input_region.ndim >= 1) + len.x = input_region.size[0]; + if (input_region.ndim >= 2) + len.y = input_region.size[1]; + if (input_region.ndim >= 3) + len.z = input_region.size[2]; + if (input_region.ndim > 3) { + LOG_ERROR("Invalid input region ndim: %d\n", input_region.ndim); + return false; + } + + psz_compressor *comp = psz_create_default(dtype, len); + + uint8_t * d_compressed = NULL; + size_t compressed_size = 0; + psz_header header; + void * record = psz_make_timerecord(); + pszerror err = + psz_compress(comp, d_data, len, 0.01, Abs, &d_compressed, &compressed_size, &header, record, 0); + if (err != PSZ_SUCCESS) { + printf("Compression failed!\n"); + return false; + } + + *region_data = (uint8_t *)malloc(compressed_size); + cudaMemcpy(*region_data, d_compressed, compressed_size, cudaMemcpyDeviceToHost); + cudaFree(d_data); + cudaFree(d_compressed); + psz_release(comp); + + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = compressed_size; + + sz_compress_params_t *out_params = (sz_compress_params_t *)PDC_malloc(sizeof(sz_compress_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("zfp_compress", PDC_TF_CPU_DEVICE, out_params, sizeof(sz_compress_params_t)); + + printf("Compressed %lu bytes\n", compressed_size); + return true; +} + +bool +pdc_tf_builtin_sz_decompress_cuda(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_sz_decompress_cuda called\n"); + + psz_header header; + psz_compressor *comp = psz_create_from_header(&header); + if (!comp) { + LOG_ERROR("Failed to create cuSZ decompressor\n"); + return false; + } + + uint8_t *d_compressed = NULL; + CUDA_CHECK(cudaMalloc((void **)&d_compressed, input_region.size[0])); + CUDA_CHECK(cudaMemcpy(d_compressed, *region_data, input_region.size[0], cudaMemcpyHostToDevice)); + + psz_len3 len = {output_region->size[0], output_region->ndim > 1 ? output_region->size[1] : 1, + output_region->ndim > 2 ? output_region->size[2] : 1}; + void * d_decompressed = NULL; + size_t nbytes = len.x * len.y * len.z * sizeof(float); + CUDA_CHECK(cudaMalloc(&d_decompressed, nbytes)); + + void * record = capi_psz_make_timerecord(); + pszerror p_err = psz_decompress(comp, d_compressed, input_region.size[0], d_decompressed, len, record, 0); + if (p_err != PSZ_SUCCESS) { + LOG_ERROR("cuSZ decompression failed\n"); + cudaFree(d_compressed); + cudaFree(d_decompressed); + psz_release(comp); + return false; + } + CUDA_CHECK(cudaDeviceSynchronize()); + + void *h_decompressed = malloc(nbytes); + CUDA_CHECK(cudaMemcpy(h_decompressed, d_decompressed, nbytes, cudaMemcpyDeviceToHost)); + *region_data = h_decompressed; + + cudaFree(d_compressed); + cudaFree(d_decompressed); + + p_err = psz_clear_buffer(comp); + if (p_err != PSZ_SUCCESS) { + LOG_ERROR("cuSZ psz_clear_buffer failed\n"); + return false; + } + p_err = psz_release(comp); + if (p_err != PSZ_SUCCESS) { + LOG_ERROR("cuSZ psz_release failed\n"); + return false; + } + + LOG_INFO("cuSZ decompression succeeded\n"); + return true; +} + +#endif // ENABLE_TF_SZ_GPU_COMPRESSSION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_turbo_cpu.c b/src/server/transform/pdc_tf_builtin_turbo_cpu.c new file mode 100644 index 000000000..7e8a7cac9 --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_turbo_cpu.c @@ -0,0 +1,95 @@ +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_TURBO_COMPRESSION +#include + +typedef struct turbo_compress_params_t { + pdc_tf_region_t decompressed_region; +} turbo_compress_params_t; + +bool +pdc_tf_builtin_turbo_compress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_turbo_compress called\n"); + PDCtf_log_pdc_region_t(input_region); + + size_t size = PDCtf_get_pdc_region_t_bytes(input_region); + void * compressed = malloc(size); + size_t ret; + + switch (input_region.pdc_var_type) { + case PDC_INT: + case PDC_INT32: + ret = p4nenc32((uint32_t *)*region_data, PDCtf_get_pdc_region_t_elements(input_region), + (unsigned char *)compressed); + break; + case PDC_INT64: + ret = p4nenc64((uint64_t *)*region_data, PDCtf_get_pdc_region_t_elements(input_region), + (unsigned char *)compressed); + break; + default: + LOG_ERROR("Invalid element type\n"); + return false; + } + + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = ret; + *region_data = compressed; + + turbo_compress_params_t *out_params = + (turbo_compress_params_t *)PDC_malloc(sizeof(turbo_compress_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("turbo_compress", PDC_TF_CPU_DEVICE, out_params, sizeof(turbo_compress_params_t)); + + LOG_DEBUG("Turbo compression succeeded: %zu bytes\n", ret); + return true; +} + +bool +pdc_tf_builtin_turbo_decompress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_turbo_decompress called\n"); + PDCtf_log_pdc_region_t(input_region); + + turbo_compress_params_t *in_params = NULL; + uint64_t in_params_size = 0; + GET_FUNC_PARAMS("turbo_compress", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + + size_t size = PDCtf_get_pdc_region_t_bytes(in_params->decompressed_region); + void * decompressed = malloc(size); + size_t ret; + + switch (in_params->decompressed_region.pdc_var_type) { + case PDC_INT: + case PDC_INT32: + ret = p4ndec32((unsigned char *)*region_data, + PDCtf_get_pdc_region_t_elements(in_params->decompressed_region), + (uint32_t *)decompressed); + break; + case PDC_INT64: + ret = p4ndec64((unsigned char *)*region_data, + PDCtf_get_pdc_region_t_elements(in_params->decompressed_region), + (uint64_t *)decompressed); + break; + default: + LOG_ERROR("Invalid element type\n"); + return false; + } + + *region_data = decompressed; + PDCtf_copy_tf_region_t(&in_params->decompressed_region, output_region); + + LOG_DEBUG("Turbo decompression succeeded: %zu bytes\n", ret); + return true; +} + +#endif // ENABLE_TF_TURBO_COMPRESSION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_zfp_cpu.c b/src/server/transform/pdc_tf_builtin_zfp_cpu.c new file mode 100644 index 000000000..67296bf75 --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_zfp_cpu.c @@ -0,0 +1,257 @@ +#include +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_ZFP_COMPRESSION +#include + +typedef struct zfp_compress_params_t { + pdc_tf_region_t decompressed_region; +} zfp_compress_params_t; + +#define FIXED_CR_RATIO 1 + +static void +print_ztype(zfp_type z_type) +{ + switch (z_type) { + case zfp_type_int32: + LOG_DEBUG("ZFP type: int32\n"); + break; + case zfp_type_int64: + LOG_DEBUG("ZFP type: int64\n"); + break; + case zfp_type_float: + LOG_DEBUG("ZFP type: float\n"); + break; + case zfp_type_double: + LOG_DEBUG("ZFP type: double\n"); + break; + case zfp_type_none: + default: + LOG_ERROR("ZFP type: none/unknown (%d)\n", (int)z_type); + break; + } +} + +static bool +pdc_tf_builtin_zfp_compress_helper(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_zfp_compress was called\n"); + PDCtf_log_pdc_region_t(input_region); + + zfp_type z_type; + switch (input_region.pdc_var_type) { + case PDC_FLOAT: + z_type = zfp_type_float; + break; + case PDC_DOUBLE: + z_type = zfp_type_double; + break; + case PDC_INT: + case PDC_INT32: + z_type = zfp_type_int32; + break; + case PDC_INT64: + z_type = zfp_type_int64; + break; + default: + LOG_ERROR("Invalid element type\n"); + return false; + } + print_ztype(z_type); + size_t bits_per_value = (8 * PDC_get_var_type_size(input_region.pdc_var_type)) / FIXED_CR_RATIO; + LOG_INFO("BITS PER VALUE %d\n", bits_per_value); + + zfp_field *field = NULL; + switch (input_region.ndim) { + case 1: + field = zfp_field_1d(*region_data, z_type, input_region.size[0]); + break; + case 2: + field = zfp_field_2d(*region_data, z_type, input_region.size[0], input_region.size[1]); + break; + case 3: + field = zfp_field_3d(*region_data, z_type, input_region.size[0], input_region.size[1], + input_region.size[2]); + break; + case 4: + field = zfp_field_4d(*region_data, z_type, input_region.size[0], input_region.size[1], + input_region.size[2], input_region.size[3]); + break; + case 0: + LOG_ERROR("ZFP compression not supported for 0 dimensions\n"); + return false; + default: + LOG_ERROR("ZFP compression not supported for > 4 dimensions\n"); + return false; + } + if (!field) { + LOG_ERROR("field was NULL\n"); + return false; + } + + zfp_stream *zfp = zfp_stream_open(NULL); + size_t bufsize = zfp_stream_maximum_size(zfp, field); + void * compressed_buffer = PDC_malloc(bufsize); + + bitstream *stream = stream_open(compressed_buffer, bufsize); + zfp_stream_set_rate(zfp, bits_per_value, z_type, input_region.ndim, 1); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + size_t compressed_size = zfp_compress(zfp, field); + + *region_data = compressed_buffer; + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = compressed_size; + + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + + zfp_compress_params_t *out_params = (zfp_compress_params_t *)PDC_malloc(sizeof(zfp_compress_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("zfp_compress", PDC_TF_CPU_DEVICE, out_params, sizeof(zfp_compress_params_t)); + + LOG_INFO("ZFP compression succeeded, compressed size: %zu bytes\n", compressed_size); + return true; +} + +static bool +pdc_tf_builtin_zfp_decompress_helper(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_zfp_decompress was called\n"); + PDCtf_log_pdc_region_t(input_region); + + zfp_compress_params_t *in_params = NULL; + uint64_t in_params_size; + + GET_FUNC_PARAMS("zfp_compress", PDC_TF_GPU_DEVICE, (void **)&in_params, &in_params_size); + if (!in_params) + GET_FUNC_PARAMS("zfp_compress", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + if (!in_params) { + LOG_ERROR("Failed to get ZFP compression parameters.\n"); + return false; + } + + PDCtf_log_pdc_region_t(in_params->decompressed_region); + + zfp_type z_type; + switch (in_params->decompressed_region.pdc_var_type) { + case PDC_FLOAT: + z_type = zfp_type_float; + break; + case PDC_DOUBLE: + z_type = zfp_type_double; + break; + case PDC_INT: + case PDC_INT32: + z_type = zfp_type_int32; + break; + case PDC_INT64: + z_type = zfp_type_int64; + break; + default: + LOG_ERROR("Invalid element type %d\n", in_params->decompressed_region.pdc_var_type); + return false; + } + print_ztype(z_type); + + size_t bits_per_value = + (8 * PDC_get_var_type_size(in_params->decompressed_region.pdc_var_type)) / FIXED_CR_RATIO; + size_t compressed_size = input_region.size[0]; + + bitstream * stream = stream_open(*region_data, compressed_size); + zfp_stream *zfp = zfp_stream_open(NULL); + zfp_stream_set_rate(zfp, bits_per_value, z_type, in_params->decompressed_region.ndim, 1); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_rewind(zfp); + + size_t total_bytes = PDCtf_get_pdc_region_t_bytes(in_params->decompressed_region); + void * buf = malloc(total_bytes); + + zfp_field *field = NULL; + switch (in_params->decompressed_region.ndim) { + case 1: + field = zfp_field_1d(buf, z_type, in_params->decompressed_region.size[0]); + break; + case 2: + field = zfp_field_2d(buf, z_type, in_params->decompressed_region.size[0], + in_params->decompressed_region.size[1]); + break; + case 3: + field = + zfp_field_3d(buf, z_type, in_params->decompressed_region.size[0], + in_params->decompressed_region.size[1], in_params->decompressed_region.size[2]); + break; + case 4: + field = zfp_field_4d( + buf, z_type, in_params->decompressed_region.size[0], in_params->decompressed_region.size[1], + in_params->decompressed_region.size[2], in_params->decompressed_region.size[3]); + break; + default: + LOG_ERROR("Unsupported ndim: %d\n", in_params->decompressed_region.ndim); + return false; + } + + size_t decompressed_size = zfp_decompress(zfp, field); + PDCtf_copy_tf_region_t(&in_params->decompressed_region, output_region); + *region_data = buf; + + size_t sizes[4]; + size_t num_elements = zfp_field_size(field, sizes); + size_t elem_size = 0; + switch (zfp_field_type(field)) { + case zfp_type_int32: + elem_size = 4; + break; + case zfp_type_int64: + elem_size = 8; + break; + case zfp_type_float: + elem_size = 4; + break; + case zfp_type_double: + elem_size = 8; + break; + default: + elem_size = 0; + break; + } + LOG_DEBUG("Actual decompressed size: %zu bytes\n", num_elements * elem_size); + + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + + return true; +} + +bool +pdc_tf_builtin_zfp_compress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + return pdc_tf_builtin_zfp_compress_helper(internal_param, params_str, region_data, input_region, + output_region); +} + +bool +pdc_tf_builtin_zfp_decompress(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + return pdc_tf_builtin_zfp_decompress_helper(internal_param, params_str, region_data, input_region, + output_region); +} + +#endif // ENABLE_TF_ZFP_COMPRESSION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_builtin_zfp_gpu.c b/src/server/transform/pdc_tf_builtin_zfp_gpu.c new file mode 100644 index 000000000..cc0d8ad75 --- /dev/null +++ b/src/server/transform/pdc_tf_builtin_zfp_gpu.c @@ -0,0 +1,304 @@ +#include +#include + +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf_common.h" +#include "pdc_tf_user.h" +#include "pdc_logger.h" + +#ifdef ENABLE_TF_ZFP_COMPRESSION +#ifdef CUDA_ENABLED + +#include +#include + +typedef struct zfp_compress_params_t { + pdc_tf_region_t decompressed_region; +} zfp_compress_params_t; + +#define FIXED_CR_RATIO 1 + +#define CUDA_CHECK(call) \ + do { \ + cudaError_t err = call; \ + if (err != cudaSuccess) { \ + fprintf(stderr, "[CUDA ERROR] %s:%d: %s (code %d)\n", __FILE__, __LINE__, \ + cudaGetErrorString(err), err); \ + exit(EXIT_FAILURE); \ + } \ + cudaDeviceSynchronize(); \ + } while (0) + +static bool +pdc_tf_builtin_zfp_compress_cuda_helper(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_zfp_compress_cuda was called\n"); + PDCtf_log_pdc_region_t(input_region); + + zfp_type z_type; + switch (input_region.pdc_var_type) { + case PDC_FLOAT: + z_type = zfp_type_float; + break; + case PDC_DOUBLE: + z_type = zfp_type_double; + break; + case PDC_INT: + case PDC_INT32: + z_type = zfp_type_int32; + break; + case PDC_INT64: + z_type = zfp_type_int64; + break; + default: + LOG_ERROR("Invalid element type\n"); + return false; + } + size_t bits_per_value = (8 * PDC_get_var_type_size(input_region.pdc_var_type)) / FIXED_CR_RATIO; + size_t num_bytes = PDCtf_get_pdc_region_t_bytes(input_region); + + void *dev_in = NULL; + void *dev_out = NULL; + CUDA_CHECK(cudaMalloc(&dev_in, num_bytes)); + + START_HOST_TO_DEV_TIME(); + CUDA_CHECK(cudaMemcpy(dev_in, *region_data, num_bytes, cudaMemcpyHostToDevice)); + END_HOST_TO_DEV_TIME(); + + zfp_field *field = NULL; + switch (input_region.ndim) { + case 1: + field = zfp_field_1d(dev_in, z_type, input_region.size[0]); + break; + case 2: + field = zfp_field_2d(dev_in, z_type, input_region.size[0], input_region.size[1]); + break; + case 3: + field = zfp_field_3d(dev_in, z_type, input_region.size[0], input_region.size[1], + input_region.size[2]); + break; + case 4: + field = zfp_field_4d(dev_in, z_type, input_region.size[0], input_region.size[1], + input_region.size[2], input_region.size[3]); + break; + default: + LOG_ERROR("Unsupported ndim: %d\n", input_region.ndim); + cudaFree(dev_in); + return false; + } + + zfp_stream *zfp = zfp_stream_open(NULL); + size_t bufsize = zfp_stream_maximum_size(zfp, field); + cudaMalloc(&dev_out, bufsize); + + bitstream *stream = stream_open(dev_out, bufsize); + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_set_rate(zfp, bits_per_value, z_type, input_region.ndim, 1); + zfp_stream_set_execution(zfp, zfp_exec_cuda); + zfp_stream_rewind(zfp); + + size_t compressed_size = zfp_compress(zfp, field); + CUDA_CHECK(cudaDeviceSynchronize()); + assert(compressed_size > 0); + + void *host_compressed = malloc(compressed_size); + + START_DEV_TO_HOST_TIME(); + CUDA_CHECK(cudaMemcpy(host_compressed, dev_out, compressed_size, cudaMemcpyDeviceToHost)); + END_DEV_TO_HOST_TIME(); + + CUDA_CHECK(cudaFree(dev_in)); + CUDA_CHECK(cudaFree(dev_out)); + + *region_data = host_compressed; + output_region->ndim = 1; + output_region->pdc_var_type = PDC_CHAR; + output_region->size[0] = compressed_size; + + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + + zfp_compress_params_t *out_params = (zfp_compress_params_t *)malloc(sizeof(zfp_compress_params_t)); + PDCtf_copy_tf_region_t(&input_region, &out_params->decompressed_region); + SET_FUNC_PARAMS("zfp_compress", PDC_TF_GPU_DEVICE, out_params, sizeof(zfp_compress_params_t)); + + LOG_DEBUG("CUDA ZFP compression succeeded, %zu bytes\n", compressed_size); + return true; +} + +static bool +pdc_tf_builtin_zfp_decompress_cuda_helper(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) +{ + LOG_DEBUG("pdc_tf_builtin_zfp_decompress_cuda_safe called\n"); + PDCtf_log_pdc_region_t(input_region); + + zfp_compress_params_t *in_params = NULL; + uint64_t in_params_size = 0; + + GET_FUNC_PARAMS("zfp_compress", PDC_TF_GPU_DEVICE, (void **)&in_params, &in_params_size); + if (!in_params) + GET_FUNC_PARAMS("zfp_compress", PDC_TF_CPU_DEVICE, (void **)&in_params, &in_params_size); + if (!in_params) { + LOG_ERROR("Failed to get ZFP compression parameters.\n"); + return false; + } + + zfp_type z_type; + size_t type_size = 0; + switch (in_params->decompressed_region.pdc_var_type) { + case PDC_FLOAT: + z_type = zfp_type_float; + type_size = sizeof(float); + break; + case PDC_DOUBLE: + z_type = zfp_type_double; + type_size = sizeof(double); + break; + case PDC_INT: + case PDC_INT32: + z_type = zfp_type_int32; + type_size = sizeof(int32_t); + break; + case PDC_INT64: + z_type = zfp_type_int64; + type_size = sizeof(int64_t); + break; + default: + LOG_ERROR("Unsupported element type %d\n", in_params->decompressed_region.pdc_var_type); + return false; + } + size_t bits_per_value = + (8 * PDC_get_var_type_size(in_params->decompressed_region.pdc_var_type)) / FIXED_CR_RATIO; + + size_t total_elements = 1; + for (int i = 0; i < in_params->decompressed_region.ndim; i++) + total_elements *= in_params->decompressed_region.size[i]; + size_t uncompressed_size = total_elements * type_size; + size_t compressed_size = input_region.size[0]; + + void *dev_compressed = NULL; + void *dev_uncompressed = NULL; + CUDA_CHECK(cudaMalloc(&dev_compressed, compressed_size + 64)); + CUDA_CHECK(cudaMalloc(&dev_uncompressed, uncompressed_size)); + + START_HOST_TO_DEV_TIME(); + CUDA_CHECK(cudaMemcpy(dev_compressed, *region_data, compressed_size, cudaMemcpyHostToDevice)); + END_HOST_TO_DEV_TIME(); + + zfp_field *field = NULL; + switch (in_params->decompressed_region.ndim) { + case 1: + field = zfp_field_1d(dev_uncompressed, z_type, in_params->decompressed_region.size[0]); + break; + case 2: + field = zfp_field_2d(dev_uncompressed, z_type, in_params->decompressed_region.size[0], + in_params->decompressed_region.size[1]); + break; + case 3: + field = + zfp_field_3d(dev_uncompressed, z_type, in_params->decompressed_region.size[0], + in_params->decompressed_region.size[1], in_params->decompressed_region.size[2]); + break; + case 4: + field = + zfp_field_4d(dev_uncompressed, z_type, in_params->decompressed_region.size[0], + in_params->decompressed_region.size[1], in_params->decompressed_region.size[2], + in_params->decompressed_region.size[3]); + break; + default: + LOG_ERROR("Unsupported ndim: %d\n", in_params->decompressed_region.ndim); + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + return false; + } + + zfp_stream *zfp = zfp_stream_open(NULL); + if (!zfp) { + LOG_ERROR("Failed to open zfp stream\n"); + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + zfp_field_free(field); + return false; + } + + bitstream *stream = stream_open(dev_compressed, compressed_size); + if (!stream) { + LOG_ERROR("Failed to open bitstream\n"); + zfp_stream_close(zfp); + zfp_field_free(field); + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + return false; + } + + zfp_stream_set_bit_stream(zfp, stream); + zfp_stream_set_rate(zfp, bits_per_value, z_type, in_params->decompressed_region.ndim, 1); + zfp_stream_set_execution(zfp, zfp_exec_cuda); + zfp_stream_rewind(zfp); + + size_t decompressed_size = zfp_decompress(zfp, field); + CUDA_CHECK(cudaDeviceSynchronize()); + if (decompressed_size == 0) { + LOG_ERROR("ZFP decompression failed\n"); + stream_close(stream); + zfp_stream_close(zfp); + zfp_field_free(field); + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + return false; + } + + void *host_buf = malloc(uncompressed_size); + if (!host_buf) { + LOG_ERROR("Failed to allocate host buffer\n"); + stream_close(stream); + zfp_stream_close(zfp); + zfp_field_free(field); + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + return false; + } + + START_DEV_TO_HOST_TIME(); + CUDA_CHECK(cudaMemcpy(host_buf, dev_uncompressed, uncompressed_size, cudaMemcpyDeviceToHost)); + END_DEV_TO_HOST_TIME(); + + CUDA_CHECK(cudaFree(dev_compressed)); + CUDA_CHECK(cudaFree(dev_uncompressed)); + + *region_data = host_buf; + PDCtf_copy_tf_region_t(&in_params->decompressed_region, output_region); + + zfp_field_free(field); + zfp_stream_close(zfp); + stream_close(stream); + + LOG_DEBUG("CUDA ZFP decompression succeeded: %zu bytes\n", decompressed_size); + return true; +} + +bool +pdc_tf_builtin_zfp_compress_cuda(pdc_tf_internal_param *internal_param, char *params_str, void **region_data, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region) +{ + return pdc_tf_builtin_zfp_compress_cuda_helper(internal_param, params_str, region_data, input_region, + output_region); +} + +bool +pdc_tf_builtin_zfp_decompress_cuda(pdc_tf_internal_param *internal_param, char *params_str, + void **region_data, pdc_tf_region_t input_region, + pdc_tf_region_t *output_region) +{ + return pdc_tf_builtin_zfp_decompress_cuda_helper(internal_param, params_str, region_data, input_region, + output_region); +} + +#endif // CUDA_ENABLED +#endif // ENABLE_TF_ZFP_COMPRESSION \ No newline at end of file diff --git a/src/server/transform/pdc_tf_common.c b/src/server/transform/pdc_tf_common.c new file mode 100644 index 000000000..acf9a8712 --- /dev/null +++ b/src/server/transform/pdc_tf_common.c @@ -0,0 +1,662 @@ +#include +#include +#include +#include +#include + +#include "common_io.h" +#include "pdc_malloc.h" +#include "pdc_tf_common.h" +#include "pdc_tf_builtin_common.h" +#include "pdc_client_server_common.h" +#include "pdc_tf.h" +#include "pdc_timing.h" +#include "pdc_interface.h" +#include "pdc_tf_poly_sched.h" +#include "json-c/json.h" + +PDC_VECTOR *pdc_tf_builtin_funcs_vector_g = NULL; + +void +append_host_to_dev_time(pdc_tf_builtin_func_t *func, double value) +{ + func->host_to_dev_avg_time[func->cur_host_to_dev_avg_time_index] = value; + func->cur_host_to_dev_avg_time_index = (func->cur_host_to_dev_avg_time_index + 1) % NUM_TF_FUNC_TIMES; +} + +void +append_dev_to_host_time(pdc_tf_builtin_func_t *func, double value) +{ + func->dev_to_host_avg_time[func->cur_dev_to_host_avg_time_index] = value; + func->cur_dev_to_host_avg_time_index = (func->cur_dev_to_host_avg_time_index + 1) % NUM_TF_FUNC_TIMES; +} + +void +append_exec_time(pdc_tf_builtin_func_t *func, double value) +{ + func->exec_avg_time[func->cur_exec_avg_time_index] = value; + func->cur_exec_avg_time_index = (func->cur_exec_avg_time_index + 1) % NUM_TF_FUNC_TIMES; +} + +double +get_host_to_dev_avg(const pdc_tf_builtin_func_t *func) +{ + double sum = 0.0; + for (int i = 0; i < NUM_TF_FUNC_TIMES; i++) + sum += func->host_to_dev_avg_time[i]; + return sum / NUM_TF_FUNC_TIMES; +} + +double +get_dev_to_host_avg(const pdc_tf_builtin_func_t *func) +{ + double sum = 0.0; + for (int i = 0; i < NUM_TF_FUNC_TIMES; i++) + sum += func->dev_to_host_avg_time[i]; + return sum / NUM_TF_FUNC_TIMES; +} + +double +get_exec_avg(const pdc_tf_builtin_func_t *func) +{ + double sum = 0.0; + for (int i = 0; i < NUM_TF_FUNC_TIMES; i++) + sum += func->exec_avg_time[i]; + return sum / NUM_TF_FUNC_TIMES; +} + +perr_t +PDCtf_set_tf_region_t(pdc_tf_region_t *dest, uint8_t ndim, pdc_var_type_t pdc_var_type, uint64_t *size) +{ + FUNC_ENTER(NULL); + + PDC_get_var_type_size(pdc_var_type); + + dest->ndim = ndim; + dest->pdc_var_type = pdc_var_type; + for (int i = 0; i < ndim; i++) + dest->size[i] = size[i]; + + FUNC_LEAVE(SUCCEED); +} + +perr_t +PDCtf_copy_tf_region_t(pdc_tf_region_t *src, pdc_tf_region_t *dest) +{ + FUNC_ENTER(NULL); + + dest->ndim = src->ndim; + dest->pdc_var_type = src->pdc_var_type; + for (int i = 0; i < src->ndim; i++) + dest->size[i] = src->size[i]; + + FUNC_LEAVE(SUCCEED); +} + +perr_t +PDCtf_add_builtin_func(char *func_name, c_func_t c_func, pdc_tf_dev_t dev) +{ + FUNC_ENTER(NULL); + + int ret_value = SUCCEED; + + if (func_name == NULL) + PGOTO_ERROR(FAIL, "func_name was NULL"); + if (c_func == NULL) + PGOTO_ERROR(FAIL, "c_func was NULL"); + + pdc_tf_builtin_func_t *builtin_func = PDC_malloc(sizeof(pdc_tf_builtin_func_t)); + pdc_vector_add(pdc_tf_builtin_funcs_vector_g, builtin_func); + + builtin_func->name = strdup(func_name); + builtin_func->c_func = c_func; + builtin_func->dev = dev; + + /* Initialize rolling history indices */ + builtin_func->cur_host_to_dev_avg_time_index = 0; + builtin_func->cur_dev_to_host_avg_time_index = 0; + builtin_func->cur_exec_avg_time_index = 0; + + /* Initialize timing histories */ + for (int i = 0; i < NUM_TF_FUNC_TIMES; i++) { + builtin_func->host_to_dev_avg_time[i] = 0.0; + builtin_func->dev_to_host_avg_time[i] = 0.0; + builtin_func->exec_avg_time[i] = (builtin_func->dev == PDC_TF_CPU_DEVICE) ? 0.750 : 0.0; + } + +done: + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_link_builtin_func(char *func_name, pdc_tf_dev_t dev, pdc_tf_func_t *f) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + bool found = false; + + if (func_name == NULL) + PGOTO_ERROR(FAIL, "func_name was NULL"); + if (f == NULL) + PGOTO_ERROR(FAIL, "f was NULL"); + + PDC_VECTOR_ITERATOR *builtin_func_iter = pdc_vector_iterator_new(pdc_tf_builtin_funcs_vector_g); + while (pdc_vector_iterator_has_next(builtin_func_iter)) { + pdc_tf_builtin_func_t *builtin_func = pdc_vector_iterator_next(builtin_func_iter); + if (builtin_func == NULL) + PGOTO_ERROR(FAIL, "builtin_func was NULL"); + if (strcmp(builtin_func->name, func_name) == 0 && builtin_func->dev == dev) { + found = true; + f->c_func = builtin_func->c_func; + } + } + pdc_vector_iterator_destroy(builtin_func_iter); + + if (!found) + PGOTO_ERROR(FAIL, "Builtin function not found"); + +done: + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_init_builtin_funcs() +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + +#ifdef CUDA_ENABLED + const char *coeff_file = getenv("PDC_POLY_COEFF_FILE"); + if (coeff_file == NULL) + coeff_file = "/pscratch/sd/n/nlewi26/src/work_space/poly_coefficients.txt"; + if (pdc_tf_poly_sched_init(coeff_file) != 0) + PGOTO_ERROR(FAIL, "Failed to initialize polynomial scheduler from %s", coeff_file); +#endif + + if (pdc_tf_builtin_funcs_vector_g == NULL) + pdc_tf_builtin_funcs_vector_g = pdc_vector_create(16, 2.0); + if (pdc_tf_builtin_funcs_vector_g == NULL) + PGOTO_ERROR(FAIL, "pdc_tf_builtin_funcs_vector_g was NULL"); +#ifdef ENABLE_TF_SZ_GPU_COMPRESSSION + if (PDCtf_add_builtin_func("sz_compress", pdc_tf_builtin_sz_compress_cuda, PDC_TF_GPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func sz_compress GPU"); + if (PDCtf_add_builtin_func("sz_decompress", pdc_tf_builtin_sz_decompress_cuda, PDC_TF_GPU_DEVICE) != + SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func sz_decompress GPU"); +#endif +#ifdef ENABLE_TF_SZ_COMPRESSION + if (PDCtf_add_builtin_func("sz_compress", pdc_tf_builtin_sz_compress, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func sz_compress CPU"); + if (PDCtf_add_builtin_func("sz_decompress", pdc_tf_builtin_sz_decompress, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func sz_decompress CPU"); +#endif +#ifdef ENABLE_TF_ZFP_COMPRESSION + if (PDCtf_add_builtin_func("zfp_compress", pdc_tf_builtin_zfp_compress, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func zfp_compress CPU"); + if (PDCtf_add_builtin_func("zfp_decompress", pdc_tf_builtin_zfp_decompress, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func zfp_decompress CPU"); +#ifdef CUDA_ENABLED + if (PDCtf_add_builtin_func("zfp_compress", pdc_tf_builtin_zfp_compress_cuda, PDC_TF_GPU_DEVICE) != + SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func zfp_compress GPU"); + if (PDCtf_add_builtin_func("zfp_decompress", pdc_tf_builtin_zfp_decompress_cuda, PDC_TF_GPU_DEVICE) != + SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func zfp_decompress GPU"); +#endif +#endif +#ifdef ENABLE_TF_SECRET_BOX_ENCRYPTION + if (PDCtf_add_builtin_func("secret_box_encrypt", pdc_tf_builtin_encrypt, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func secret_box_encrypt CPU"); + if (PDCtf_add_builtin_func("secret_box_decrypt", pdc_tf_builtin_decrypt, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func secret_box_decrypt CPU"); +#endif +#ifdef ENABLE_TF_TURBO_COMPRESSION + if (PDCtf_add_builtin_func("turbo_compress", pdc_tf_builtin_turbo_compress, PDC_TF_CPU_DEVICE) != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func turbo_compress CPU"); + if (PDCtf_add_builtin_func("turbo_decompress", pdc_tf_builtin_turbo_decompress, PDC_TF_CPU_DEVICE) != + SUCCEED) + PGOTO_ERROR(FAIL, "Failed to add builtin func decompress CPU"); +#endif + +done: + FUNC_LEAVE(ret_value); +} + +bool +PDCtf_region_has_attached_graph(struct pdc_tf_obj_t *tf_obj, int ndim, size_t unit, uint64_t *offset, + uint64_t *size, pdc_tf_region_mapping_t **region_mapping) +{ + FUNC_ENTER(NULL); + + bool ret_value = false; + PDC_VECTOR_ITERATOR *region_mappings_iter = NULL; + + if (tf_obj == NULL) { + LOG_DEBUG("tf_obj is NULL\n"); + PGOTO_DONE(false); + } + if (tf_obj->region_mappings_vector == NULL) { + LOG_DEBUG("region_mappings_vector is NULL\n"); + PGOTO_DONE(false); + } + + region_mappings_iter = pdc_vector_iterator_new(tf_obj->region_mappings_vector); + while (pdc_vector_iterator_has_next(region_mappings_iter)) { + *region_mapping = pdc_vector_iterator_next(region_mappings_iter); + pdc_tf_region_t *conceptual_region = &((*region_mapping)->conceptual_region); + uint64_t * conceptual_offset = (*region_mapping)->conceptual_offset; + + bool ndim_matches = conceptual_region->ndim == ndim; + bool unit_matches = PDC_get_var_type_size(conceptual_region->pdc_var_type) == unit; + bool offset_matches = true; + bool size_matches = true; + + for (int i = 0; i < ndim; i++) { + bool offset_i_match = (conceptual_offset[i] == offset[i]); + bool size_i_match = (conceptual_region->size[i] == size[i]); + offset_matches &= offset_i_match; + size_matches &= size_i_match; + } + + if (ndim_matches && offset_matches && size_matches && unit_matches) { + PGOTO_DONE(true); + } + } + +done: + if (region_mappings_iter != NULL) + pdc_vector_iterator_destroy(region_mappings_iter); + FUNC_LEAVE(ret_value); +} + +static struct array_list * +get_json_array(struct json_object *json_obj, char *arr_name) +{ + FUNC_ENTER(NULL); + + struct json_object *ret_value = NULL; + + if (json_object_object_get_ex(json_obj, arr_name, &ret_value) == 0) + PGOTO_ERROR(NULL, "%s was not found", arr_name); + if (json_object_get_type(ret_value) != json_type_array) + PGOTO_ERROR(NULL, "%s was not an array", arr_name); + +done: + if (ret_value == NULL) + FUNC_LEAVE(NULL); + FUNC_LEAVE(json_object_get_array(ret_value)); +} + +static const char * +get_json_string(struct json_object *json_obj, char *str_name, bool expect_string) +{ + struct json_object *str_json_obj = NULL; + const char * ret_value = NULL; + + if (!json_object_object_get_ex(json_obj, str_name, &str_json_obj)) { + if (expect_string) + PGOTO_ERROR(NULL, "%s was not found", str_name); + else + PGOTO_DONE(NULL); + } + if (json_object_get_type(str_json_obj) != json_type_string) + PGOTO_ERROR(NULL, "%s was not a string\n", str_json_obj); + ret_value = json_object_get_string(str_json_obj); + +done: + if (ret_value == NULL) + FUNC_LEAVE(ret_value); + FUNC_LEAVE(json_object_get_string(str_json_obj)); +} + +char *pdc_tf_dev_strs[] = {"CPU", "GPU"}; +char *pdc_tf_location_strs[] = {"builtin", "external"}; + +bool +vertices_are_equal(void *v1, void *v2) +{ + pdc_tf_state_t *s1 = (pdc_tf_state_t *)v1; + pdc_tf_state_t *s2 = (pdc_tf_state_t *)v2; + + if (s1 == NULL || s2 == NULL) + return false; + + return !strcmp(s1->name, s2->name); +} + +static void +graph_free(void *data) +{ + FUNC_ENTER(NULL); + char *json_filepath = (char *)data; + json_filepath = PDC_free(data); + FUNC_LEAVE_VOID(); +} + +static void +edge_free(void *data) +{ + FUNC_ENTER(NULL); + + pdc_tf_func_t * f = (pdc_tf_func_t *)data; + PDC_VECTOR_ITERATOR *dg_params_iter = pdc_vector_iterator_new(f->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(dg_params_iter); + if (cur_param != NULL) + cur_param = PDC_free(cur_param); + } + pdc_vector_iterator_destroy(dg_params_iter); + if (f->params_str != NULL) + f->params_str = PDC_free(f->params_str); + f->name = PDC_free(f->name); + f = PDC_free(f); + + FUNC_LEAVE_VOID(); +} + +static void +vertex_free(void *data) +{ + FUNC_ENTER(NULL); + + pdc_tf_state_t * s = (pdc_tf_state_t *)data; + PDC_VECTOR_ITERATOR *dg_params_iter = pdc_vector_iterator_new(s->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_tf_dg_params_t *cur_param = pdc_vector_iterator_next(dg_params_iter); + if (cur_param != NULL) + cur_param = PDC_free(cur_param); + } + pdc_vector_iterator_destroy(dg_params_iter); + s->name = PDC_free(s->name); + s = PDC_free(s); + + FUNC_LEAVE_VOID(); +} + +pdc_dg_t * +PDCtf_dg_json_create_common(char *filepath) +{ + FUNC_ENTER(NULL); + + pdc_dg_t * ret_value = NULL; + pdc_dg_t * dg_cpy = NULL; + FILE * fp = NULL; + struct json_object *json_obj = NULL; + io_buffer_t io_buffer; + memset(&io_buffer, 0, sizeof(io_buffer_t)); + + if ((fp = open_file(filepath, IO_MODE_READ)) == NULL) + PGOTO_ERROR(0, "Failed to open_file: %s\n", filepath); + if (read_file(fp, &io_buffer) != 0) + PGOTO_ERROR(NULL, "Failed to read_file"); + + if ((json_obj = json_tokener_parse(io_buffer.buffer)) == NULL) + PGOTO_ERROR(NULL, "Failed to parse JSON"); + + const char *dg_name = get_json_string(json_obj, "name", true); + if (dg_name == NULL) + PGOTO_ERROR(NULL, "Failed to find graph name"); + const char *lib_path = NULL; + if ((lib_path = get_json_string(json_obj, "lib_path", false)) != NULL) + LOG_DEBUG("Library path: %s\n", lib_path); + + ret_value = PDCdg_create(graph_free, vertices_are_equal, NULL, edge_free, vertex_free); + dg_cpy = ret_value; + (ret_value)->data = strdup(filepath); + + struct array_list *states = get_json_array(json_obj, "states"); + struct array_list *functions = get_json_array(json_obj, "functions"); + if (states == NULL || functions == NULL) + PGOTO_DONE(NULL); + + int states_length = array_list_length(states); + for (int i = 0; i < states_length; i++) { + struct json_object *s = array_list_get_idx(states, i); + char * s_name = strdup(get_json_string(s, "name", true)); + + if (s_name == NULL) + PGOTO_DONE(NULL); + + pdc_tf_state_t *dg_state = PDC_calloc(1, sizeof(pdc_tf_state_t)); + dg_state->name = s_name; + + if (PDCdg_add_vertex(ret_value, dg_state) == PDC_DG_INVALID_VERTEX) + PGOTO_ERROR(NULL, "Failed to add vertex to directed graph"); + } + + int functions_length = array_list_length(functions); + for (int i = 0; i < functions_length; i++) { + struct json_object *f = array_list_get_idx(functions, i); + + char *f_name = strdup(get_json_string(f, "name", true)); + char *f_input_state = strdup(get_json_string(f, "input_state", true)); + char *f_output_state = strdup(get_json_string(f, "output_state", true)); + char *f_params_str = NULL; + if (get_json_string(f, "params", false) != NULL) + f_params_str = strdup(get_json_string(f, "params", false)); + const char *f_device = get_json_string(f, "device", true); + const char *f_location = get_json_string(f, "location", true); + + if (f_name == NULL || f_input_state == NULL || f_output_state == NULL || f_location == NULL) + PGOTO_DONE(NULL); + + pdc_tf_func_t *dg_func = PDC_calloc(1, sizeof(pdc_tf_func_t)); + + pdc_tf_dev_t dev; + bool found_device = false; + for (int j = 0; j < PDC_TF_NUM_DEVICES; j++) { + if (!strcmp(f_device, pdc_tf_dev_strs[j])) { + found_device = true; + dev = j; + break; + } + } + if (!found_device) + PGOTO_ERROR(NULL, "Invalid device %s\n", f_device); + + pdc_tf_location_t location; + bool found_location = false; + for (int j = 0; j < PDC_TF_NUM_LOCATIONS; j++) { + if (!strcmp(f_location, pdc_tf_location_strs[j])) { + found_location = true; + location = j; + break; + } + } + if (!found_location) + PGOTO_ERROR(NULL, "Invalid location %s\n", f_location); + + if (location == PDC_TF_EXTERNAL) { + if (lib_path == NULL) + PGOTO_ERROR(NULL, "Function %s is external but no lib_path was provided\n", f_name); + + void *handle = dlopen(lib_path, RTLD_LAZY); + if (!handle) + PGOTO_ERROR(NULL, "Failed to dlopen library at path %s: %s\n", lib_path, dlerror()); + + dlerror(); + void *func_ptr = dlsym(handle, f_name); + char *error; + if ((error = dlerror()) != NULL) + PGOTO_ERROR(NULL, "Failed to find symbol %s in library %s: %s\n", f_name, lib_path, error); + + dg_func->c_func = func_ptr; + + if (PDCtf_add_builtin_func(f_name, dg_func->c_func, dev) != SUCCEED) + PGOTO_ERROR(NULL, "Failed to add external function %s to builtin functions vector\n", f_name); + } + + dg_func->dev = dev; + dg_func->location = location; + if (PDCtf_link_builtin_func(f_name, dev, dg_func) != SUCCEED) + PGOTO_ERROR(NULL, "Failed to link to builtin function\n"); + dg_func->name = f_name; + dg_func->params_str = (f_params_str) ? f_params_str : NULL; + + pdc_tf_state_t i_state = {.name = (char *)f_input_state}; + pdc_tf_state_t o_state = {.name = (char *)f_output_state}; + + if (PDCdg_add_edge(ret_value, &i_state, &o_state, dg_func) == PDC_DG_INVALID_EDGE) + PGOTO_ERROR(NULL, "Failed to add edge to directed graph\n"); + } + +done: + if (fp != NULL) + close_file(fp); + if (io_buffer.buffer != NULL) + PDC_free(io_buffer.buffer); + if (json_obj != NULL) + json_object_put(json_obj); + if (ret_value == NULL && dg_cpy != NULL) { + LOG_ERROR("Failed load JSON freeing graph\n"); + PDCdg_destroy(dg_cpy); + } + + FUNC_LEAVE(ret_value); +} + +size_t +PDCtf_get_pdc_region_t_elements(pdc_tf_region_t reg) +{ + FUNC_ENTER(NULL); + + size_t num_elements = 1; + for (int i = 0; i < reg.ndim; ++i) + num_elements *= reg.size[i]; + + FUNC_LEAVE(num_elements); +} + +size_t +PDCtf_get_flat_conceptual_offset(int ndim, uint64_t offset[4], const uint64_t *dims) +{ + FUNC_ENTER(NULL); + + assert(ndim > 0); + + size_t flat_offset = 0; + size_t stride = 1; + + for (int i = 0; i < ndim; i++) { + flat_offset += offset[i] * stride; + stride *= dims[i]; + } + + FUNC_LEAVE(flat_offset); +} + +size_t +PDCtf_get_pdc_region_t_bytes(pdc_tf_region_t reg) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(PDCtf_get_pdc_region_t_elements(reg) * PDC_get_var_type_size(reg.pdc_var_type)); +} + +void +PDCtf_log_pdc_region_t(pdc_tf_region_t reg) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE_VOID(); +} + +void +PDCtf_print_exec_path_common(pdc_dg_t *dg, char *cur_state, char *desired_state) +{ + FUNC_ENTER(NULL); + + assert(dg != NULL); + assert(cur_state != NULL); + assert(desired_state != NULL); + + pdc_tf_state_t tf_cur_state = {.name = cur_state}; + pdc_tf_state_t tf_desired_state = {.name = desired_state}; + pdc_dg_edge_t *edges_out; + uint32_t num_edges; + + if (PDCdg_shortest_path(dg, &tf_cur_state, &tf_desired_state, &edges_out, &num_edges)) { + for (uint32_t j = 0; j < num_edges; j++) { + pdc_dg_edge_t e = edges_out[j]; + pdc_tf_state_t *v1 = (pdc_tf_state_t *)(dg->vertices[e.v1_id]->data); + pdc_tf_state_t *v2 = (pdc_tf_state_t *)(dg->vertices[e.v2_id]->data); + } + } + else + LOG_DEBUG("No path found\n"); + + FUNC_LEAVE_VOID(); +} + +void +PDCtf_print_dg_common(pdc_dg_t *dg, bool write_to_file) +{ + int stdout_fd; + int file_fd; + + if (write_to_file) { + stdout_fd = dup(STDOUT_FILENO); + if (stdout_fd == -1) { + perror("dup"); + return; + } + + file_fd = open("graph.txt", O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (file_fd == -1) { + perror("open"); + close(stdout_fd); + return; + } + + if (dup2(file_fd, STDOUT_FILENO) == -1) { + perror("dup2"); + close(stdout_fd); + close(file_fd); + return; + } + + close(file_fd); + } + + LOG_JUST_PRINT("\tdigraph G {\n"); + LOG_JUST_PRINT("legend [shape=none, margin=0, label=<\n"); + LOG_JUST_PRINT(" \n"); + LOG_JUST_PRINT(" \n"); + LOG_JUST_PRINT(" \n"); + LOG_JUST_PRINT("
→CPU
→GPU
\n"); + LOG_JUST_PRINT(">];\n"); + LOG_JUST_PRINT("\tlabel=\"Transformation Graph\";\n"); + LOG_JUST_PRINT("\tlabelloc=top;\n"); + LOG_JUST_PRINT("\tfontsize=8;\n"); + LOG_JUST_PRINT("\tfontname=\"Helvetica\";\n"); + LOG_JUST_PRINT("\tnodesep=1.0;\n"); + LOG_JUST_PRINT("\tranksep=1.0;\n"); + LOG_JUST_PRINT("\tsplines=true;\n"); + LOG_JUST_PRINT("\toverlap=false;\n"); + LOG_JUST_PRINT("\trankdir=LR;\n"); + LOG_JUST_PRINT( + "\tnode [fontsize=10, shape=box, style=filled, fillcolor=lightgray, fontname=\"Helvetica\"];\n"); + LOG_JUST_PRINT("\tedge [fontsize=10];\n"); + + for (int i = 0; i < dg->edge_count; i++) { + pdc_dg_edge_t *edge = dg->edges[i]; + + pdc_tf_state_t *input_state = (pdc_tf_state_t *)PDCdg_get_vertex_data(dg, edge->v1_id); + pdc_tf_state_t *output_state = (pdc_tf_state_t *)PDCdg_get_vertex_data(dg, edge->v2_id); + pdc_tf_func_t * edge_func = (pdc_tf_func_t *)edge->data; + + const char *color = (edge_func->dev == PDC_TF_CPU_DEVICE) ? "blue" : "red"; + + LOG_JUST_PRINT("\t\"%s\" -> \"%s\" [label=\"%s\", color=%s];\n", input_state->name, + output_state->name, edge_func->name, color); + } + + LOG_JUST_PRINT("}\n"); + + if (write_to_file) { + fflush(stdout); + if (dup2(stdout_fd, STDOUT_FILENO) == -1) + perror("dup2 restore"); + close(stdout_fd); + } +} diff --git a/src/server/transform/pdc_tf_poly_sched.c b/src/server/transform/pdc_tf_poly_sched.c new file mode 100644 index 000000000..259d2bb16 --- /dev/null +++ b/src/server/transform/pdc_tf_poly_sched.c @@ -0,0 +1,160 @@ +/* + * pdc_tf_poly_sched.c + * + * Model: degree-3 polynomial, 4 features, 35 terms + * Features: data_size_mb, nvml_gpu_util, prev_h2d_ms, prev_d2h_ms + * Target: total_ms (H2D + compression + D2H) + * R²=0.9723, MAE=2.428 ms + */ + +#include +#include +#include +#include + +#include "pdc_tf_poly_sched.h" +#include "pdc_tf_profiler.h" +#include "pdc_logger.h" + +pdc_tf_poly_model_t pdc_tf_poly_model = {.initialized = 0}; + +int +pdc_tf_poly_sched_init(const char *coeff_file) +{ + FILE *f = fopen(coeff_file, "r"); + if (!f) { + LOG_ERROR("pdc_tf_poly_sched_init: cannot open %s\n", coeff_file); + return -1; + } + + int n_terms = 0; + if (fscanf(f, "%d", &n_terms) != 1 || n_terms <= 0 || n_terms > PDC_POLY_MAX_TERMS) { + LOG_ERROR("pdc_tf_poly_sched_init: invalid n_terms=%d in %s\n", n_terms, coeff_file); + fclose(f); + return -1; + } + + pdc_tf_poly_model.n_terms = n_terms; + + for (int t = 0; t < n_terms; t++) { + int p[PDC_POLY_N_FEATURES]; + double coeff; + int nread = 0; + for (int i = 0; i < PDC_POLY_N_FEATURES; i++) + nread += (fscanf(f, "%d", &p[i]) == 1) ? 1 : 0; + nread += (fscanf(f, "%lf", &coeff) == 1) ? 1 : 0; + + if (nread != PDC_POLY_N_FEATURES + 1) { + LOG_ERROR("pdc_tf_poly_sched_init: parse error at term %d\n", t); + fclose(f); + return -1; + } + for (int i = 0; i < PDC_POLY_N_FEATURES; i++) + pdc_tf_poly_model.powers[t][i] = p[i]; + pdc_tf_poly_model.coefficients[t] = coeff; + } + + fclose(f); + pdc_tf_poly_model.initialized = 1; + LOG_WARNING("pdc_tf_poly_sched_init: loaded %d polynomial terms from %s\n", n_terms, coeff_file); + return 0; +} + +void +pdc_tf_poly_sched_finalize(void) +{ + pdc_tf_poly_model.initialized = 0; +} + +double +pdc_tf_poly_predict(const double features[PDC_POLY_N_FEATURES]) +{ + if (!pdc_tf_poly_model.initialized) + return -1.0; + + double f[PDC_POLY_N_FEATURES]; + for (int i = 0; i < PDC_POLY_N_FEATURES; i++) + f[i] = (features[i] < 0.0) ? 0.0 : features[i]; + + double result = 0.0; + for (int t = 0; t < pdc_tf_poly_model.n_terms; t++) { + double term = pdc_tf_poly_model.coefficients[t]; + for (int i = 0; i < PDC_POLY_N_FEATURES; i++) { + int exp = pdc_tf_poly_model.powers[t][i]; + if (exp == 0) + continue; + double base = f[i]; + for (int e = 0; e < exp; e++) + term *= base; + } + result += term; + } + return (result < 5.0) ? 5.0 : result; +} + +int +pdc_tf_poly_select_gpu(double data_size_mb) +{ + unsigned int n_devs = pdc_tf_profiler_nvml_device_count; + if (n_devs == 0) + return 0; + + if (!pdc_tf_poly_model.initialized) { + LOG_WARNING("pdc_tf_poly_select_gpu: model not initialized, " + "falling back to lowest utilization\n"); + double best_util = 2.0; + unsigned int best_dev = 0; + for (unsigned int i = 0; i < n_devs; i++) { + double u = pdc_tf_avg_gpu_utilization(i); + if (u < best_util) { + best_util = u; + best_dev = i; + } + } + return (int)best_dev; + } + + double best_pred = 1e18; + unsigned int best_dev = 0; + + for (unsigned int i = 0; i < n_devs; i++) { + double prev_h2d_ms, prev_comp_ms, prev_d2h_ms, prev_total_ms; + pdc_tf_get_device_lag(i, &prev_h2d_ms, &prev_comp_ms, &prev_d2h_ms, &prev_total_ms); + + /* feature order must match export_coefficients.py FEATURES list: + * [0] data_size_mb + * [1] nvml_gpu_util (0-100 scale) + * [2] prev_h2d_ms + * [3] prev_d2h_ms + */ + double features[PDC_POLY_N_FEATURES]; + features[0] = data_size_mb; + features[1] = pdc_tf_avg_gpu_utilization(i) * 100.0; + features[2] = prev_h2d_ms; + features[3] = prev_d2h_ms; + + double pred = pdc_tf_poly_predict(features); + /* floor cold-start GPUs (no lag data yet) at 5ms so they + * don't appear artificially cheaper than warmed-up GPUs */ + if (prev_h2d_ms < 0.0 || prev_d2h_ms < 0.0) + pred = fmax(pred, 5.0); + + LOG_WARNING("pdc_tf_poly_select_gpu: GPU %u data=%.1f MB util=%.1f%% " + "prev_h2d=%.2f ms prev_d2h=%.2f ms predicted=%.2f ms\n", + i, data_size_mb, features[1], prev_h2d_ms, prev_d2h_ms, pred); + + if (pred < best_pred) { + best_pred = pred; + best_dev = i; + } + } + + LOG_WARNING("pdc_tf_poly_select_gpu: selected GPU %u (predicted %.2f ms)\n", best_dev, best_pred); + return (int)best_dev; +} + +void +pdc_tf_poly_update(unsigned int device_index, double h2d_ms, double comp_ms, double d2h_ms, double total_ms) +{ + pdc_tf_update_device_lag(device_index, h2d_ms, comp_ms, d2h_ms, total_ms); +} \ No newline at end of file diff --git a/src/server/transform/pdc_tf_profiler.c b/src/server/transform/pdc_tf_profiler.c new file mode 100644 index 000000000..ccffcba35 --- /dev/null +++ b/src/server/transform/pdc_tf_profiler.c @@ -0,0 +1,266 @@ +#include "pdc_tf_profiler.h" +#include + +int pdc_tf_profiler_init = 0; +int pdc_tf_profiler_nvml_init = 0; +int pdc_tf_profiler_cpu_init = 0; +unsigned int pdc_tf_profiler_nvml_device_count = 0; +pdc_tf_profiler_samples_t pdc_tf_profiler_samples = {0}; +static pthread_mutex_t profiler_lock = PTHREAD_MUTEX_INITIALIZER; + +/* ── GPU utilization query ────────────────────────────────────────────────── */ +double +pdc_tf_avg_gpu_utilization(unsigned int device_index) +{ + if (device_index >= pdc_tf_profiler_samples.nvml_device_count) + return -1.0; + + int count = 0; + double sum = 0.0; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) { + pdc_tf_profiler_nvml_sample_t *samples = pdc_tf_profiler_samples.nvml_samples[i]; + if (samples != NULL) { + sum += samples[device_index].gpu_utilization; + count++; + } + } + double res = (count > 0) ? (sum / count) : 0.0; + if (res < 0.0 || res > 1.0) + LOG_ERROR("Average GPU utilization out of expected range: %f\n", res); + return res; +} + +/* ── CPU utilization query ────────────────────────────────────────────────── */ +double +pdc_tf_avg_cpu_utilization(void) +{ + int count = 0; + double sum = 0.0; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) { + pdc_tf_profiler_cpu_sample_t *sample = pdc_tf_profiler_samples.cpu_samples[i]; + if (sample != NULL) { + sum += sample->cpu_utilization; + count++; + } + } + double res = (count > 0) ? (sum / count) : 0.0; + if (res < 0.0 || res > 1.0) + LOG_ERROR("Average CPU utilization out of expected range: %f\n", res); + return res; +} + +/* ── NVML profiler update — non-static so pdc_tf_server.c can call it ─────── */ +perr_t +pdc_tf_nvml_profiler_update(void) +{ + FUNC_ENTER(NULL); + perr_t ret_value = SUCCEED; + nvmlReturn_t nvml_ret; + + if (!pdc_tf_profiler_nvml_init) { + pdc_tf_profiler_nvml_init = 1; + + nvml_ret = nvmlInit(); + if (nvml_ret != NVML_SUCCESS) { + LOG_ERROR("Failed to initialize NVML: %s\n", nvmlErrorString(nvml_ret)); + PGOTO_ERROR(FAIL, "Failed to initialize NVML"); + } + + nvml_ret = nvmlDeviceGetCount(&pdc_tf_profiler_nvml_device_count); + if (nvml_ret != NVML_SUCCESS) { + LOG_ERROR("Failed to get NVML device count: %s\n", nvmlErrorString(nvml_ret)); + PGOTO_ERROR(FAIL, "Failed to get NVML device count"); + } + + pdc_tf_profiler_samples.nvml_device_count = pdc_tf_profiler_nvml_device_count; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) + pdc_tf_profiler_samples.nvml_samples[i] = NULL; + } + + if (pdc_tf_profiler_nvml_device_count == 0) + goto done; + + pdc_tf_profiler_nvml_sample_t *nvml_sample = (pdc_tf_profiler_nvml_sample_t *)PDC_malloc( + sizeof(pdc_tf_profiler_nvml_sample_t) * pdc_tf_profiler_nvml_device_count); + + for (int i = 0; i < (int)pdc_tf_profiler_nvml_device_count; i++) { + nvmlDevice_t device; + nvml_ret = nvmlDeviceGetHandleByIndex(i, &device); + if (nvml_ret != NVML_SUCCESS) + continue; + + nvmlMemory_t mem_info; + nvml_ret = nvmlDeviceGetMemoryInfo(device, &mem_info); + if (nvml_ret != NVML_SUCCESS) + continue; + + nvmlUtilization_t util_info; + nvml_ret = nvmlDeviceGetUtilizationRates(device, &util_info); + if (nvml_ret != NVML_SUCCESS) + continue; + + nvml_sample[i].gpu_utilization = (double)util_info.gpu / 100.0; + nvml_sample[i].memory_utilization = util_info.memory; + nvml_sample[i].memory_total = mem_info.total; + nvml_sample[i].memory_used = mem_info.used; + nvml_sample[i].memory_free = mem_info.free; + + unsigned int power_mw = 0; + nvmlDeviceGetPowerUsage(device, &power_mw); + nvml_sample[i].power_mw = power_mw; + + LOG_DEBUG("NVML device %d: util=%.1f%% power=%u mW\n", i, (double)util_info.gpu, power_mw); + } + + /* with buffer size 1, always write to slot 0 */ + int idx = pdc_tf_profiler_samples.nvml_head % PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; + if (pdc_tf_profiler_samples.nvml_samples[idx] != NULL) + PDC_free(pdc_tf_profiler_samples.nvml_samples[idx]); + pdc_tf_profiler_samples.nvml_samples[idx] = nvml_sample; + pdc_tf_profiler_samples.nvml_head++; + +done: + FUNC_LEAVE(ret_value); +} + +/* ── CPU profiler update ──────────────────────────────────────────────────── */ +static perr_t +pdc_tf_cpu_profiler_update(double elapsed_total_time_sec, double elapsed_progress_time_sec) +{ + FUNC_ENTER(NULL); + perr_t ret_value = SUCCEED; + + if (!pdc_tf_profiler_cpu_init) { + pdc_tf_profiler_cpu_init = 1; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) + pdc_tf_profiler_samples.cpu_samples[i] = NULL; + } + + double cpu_util = 1.0 - (elapsed_progress_time_sec / elapsed_total_time_sec); + if (cpu_util < 0.0 || cpu_util > 1.0) + LOG_ERROR("Computed CPU utilization out of expected range: %f\n", cpu_util); + + pdc_tf_profiler_cpu_sample_t *cpu_sample = + (pdc_tf_profiler_cpu_sample_t *)PDC_malloc(sizeof(pdc_tf_profiler_cpu_sample_t)); + cpu_sample->cpu_utilization = cpu_util; + + int idx = pdc_tf_profiler_samples.cpu_head % PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; + if (pdc_tf_profiler_samples.cpu_samples[idx] != NULL) + PDC_free(pdc_tf_profiler_samples.cpu_samples[idx]); + pdc_tf_profiler_samples.cpu_samples[idx] = cpu_sample; + pdc_tf_profiler_samples.cpu_head++; + + FUNC_LEAVE(ret_value); +} + +/* ── main update — called from server loop ───────────────────────────────── */ +perr_t +pdc_tf_update_profiler(double elapsed_total_time_sec, double elapsed_progress_time_sec) +{ + FUNC_ENTER(NULL); + perr_t ret_value = SUCCEED; + + if (!pdc_tf_profiler_init) { + pdc_tf_profiler_init = 1; + pdc_tf_profiler_samples.cpu_head = 0; + pdc_tf_profiler_samples.nvml_head = 0; + } + + ret_value = pdc_tf_nvml_profiler_update(); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to update NVML profiler"); + + ret_value = pdc_tf_cpu_profiler_update(elapsed_total_time_sec, elapsed_progress_time_sec); + if (ret_value != SUCCEED) + PGOTO_ERROR(FAIL, "Failed to update CPU profiler"); + +done: + FUNC_LEAVE(ret_value); +} + +/* ── per-device lag storage ──────────────────────────────────────────────── */ +#define PDC_TF_MAX_GPU_DEVICES 8 + +typedef struct { + double prev_h2d_ms; + double prev_comp_ms; + double prev_d2h_ms; + double prev_total_ms; +} pdc_tf_device_lag_t; + +static pdc_tf_device_lag_t pdc_tf_device_lag[PDC_TF_MAX_GPU_DEVICES]; +static int pdc_tf_device_lag_initialized = 0; + +static void +pdc_tf_lag_init(void) +{ + for (int i = 0; i < PDC_TF_MAX_GPU_DEVICES; i++) { + pdc_tf_device_lag[i].prev_h2d_ms = -1.0; + pdc_tf_device_lag[i].prev_comp_ms = -1.0; + pdc_tf_device_lag[i].prev_d2h_ms = -1.0; + pdc_tf_device_lag[i].prev_total_ms = -1.0; + } + pdc_tf_device_lag_initialized = 1; +} + +void +pdc_tf_update_device_lag(unsigned int device_index, double h2d_ms, double comp_ms, double d2h_ms, + double total_ms) +{ + if (!pdc_tf_device_lag_initialized) + pdc_tf_lag_init(); + if (device_index >= PDC_TF_MAX_GPU_DEVICES) + return; + pdc_tf_device_lag[device_index].prev_h2d_ms = h2d_ms; + pdc_tf_device_lag[device_index].prev_comp_ms = comp_ms; + pdc_tf_device_lag[device_index].prev_d2h_ms = d2h_ms; + pdc_tf_device_lag[device_index].prev_total_ms = total_ms; +} + +void +pdc_tf_get_device_lag(unsigned int device_index, double *prev_h2d_ms, double *prev_comp_ms, + double *prev_d2h_ms, double *prev_total_ms) +{ + if (!pdc_tf_device_lag_initialized) + pdc_tf_lag_init(); + if (device_index >= PDC_TF_MAX_GPU_DEVICES || device_index >= pdc_tf_profiler_nvml_device_count) { + *prev_h2d_ms = *prev_comp_ms = *prev_d2h_ms = *prev_total_ms = -1.0; + return; + } + *prev_h2d_ms = pdc_tf_device_lag[device_index].prev_h2d_ms; + *prev_comp_ms = pdc_tf_device_lag[device_index].prev_comp_ms; + *prev_d2h_ms = pdc_tf_device_lag[device_index].prev_d2h_ms; + *prev_total_ms = pdc_tf_device_lag[device_index].prev_total_ms; +} + +double +pdc_tf_avg_gpu_power_mw(unsigned int device_index) +{ + if (device_index >= pdc_tf_profiler_nvml_device_count) + return 0.0; + double sum = 0.0; + int count = 0; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) { + if (pdc_tf_profiler_samples.nvml_samples[i] == NULL) + continue; + sum += (double)pdc_tf_profiler_samples.nvml_samples[i][device_index].power_mw; + count++; + } + return (count > 0) ? (sum / count) : 0.0; +} + +unsigned long +pdc_tf_avg_gpu_mem_used(unsigned int device_index) +{ + if (device_index >= pdc_tf_profiler_nvml_device_count) + return 0UL; + double sum = 0.0; + int count = 0; + for (int i = 0; i < PDC_TF_PROFILE_SAMPLE_VECTOR_MAX_SIZE; i++) { + if (pdc_tf_profiler_samples.nvml_samples[i] == NULL) + continue; + sum += (double)pdc_tf_profiler_samples.nvml_samples[i][device_index].memory_used; + count++; + } + return (count > 0) ? (unsigned long)(sum / count) : 0UL; +} \ No newline at end of file diff --git a/src/server/transform/pdc_tf_server.c b/src/server/transform/pdc_tf_server.c new file mode 100644 index 000000000..47f06c39b --- /dev/null +++ b/src/server/transform/pdc_tf_server.c @@ -0,0 +1,680 @@ +#include +#include +#include + +#include "pdc_tf_server.h" +#include "pdc_malloc.h" +#include "pdc_client_server_common.h" +#include "pdc_vector.h" +#include "pdc_tf_user.h" +#include "pdc_tf_profiler.h" +#include "pdc_server_region_cache.h" +#include "pdc_tf_poly_sched.h" + +PDC_VECTOR *tf_obj_id_to_dg_vector_g = NULL; + +#ifndef IS_PDC_SERVER +perr_t +PDCtf_store_json_mapping(pdcid_t obj_id, char *json_filepath, char *cur_state, char *client_state, + char *store_state, uint64_t *offset, uint64_t *size, uint8_t ndim, + pdc_var_type_t pdc_var_type) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(SUCCEED); +} +perr_t +PDCtf_exec_graph(pdc_dg_t *dg, uint64_t flat_conceptual_offset, char *cur_state, char *desired_state, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region, void **input, int is_write, + pdc_tf_sched_mode_t sched_mode) +{ + FUNC_ENTER(NULL); + FUNC_LEAVE(SUCCEED); +} +#else +static pdcid_t cur_graph = 1; + +perr_t +PDCtf_store_json_mapping(pdcid_t obj_id, char *json_filepath, char *cur_state, char *client_state, + char *store_state, uint64_t *offset, uint64_t *size, uint8_t ndim, + pdc_var_type_t pdc_var_type) +{ + FUNC_ENTER(NULL); + + LOG_DEBUG("PDCtf_store_json_mapping was called\n"); + + perr_t ret_value = SUCCEED; + + if (tf_obj_id_to_dg_vector_g == NULL) + tf_obj_id_to_dg_vector_g = pdc_vector_create(8, 2.0); + if (tf_obj_id_to_dg_vector_g == NULL) + PGOTO_ERROR(FAIL, "tf_obj_id_to_dg_vector_g was NULL"); + + // Find object in mapping if it exists + pdc_tf_obj_id_to_dg_t *obj_id_to_dg = NULL; + PDC_VECTOR_ITERATOR * tf_obj_id_to_dg_vector_iter = pdc_vector_iterator_new(tf_obj_id_to_dg_vector_g); + while (pdc_vector_iterator_has_next(tf_obj_id_to_dg_vector_iter)) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = + (pdc_tf_obj_id_to_dg_t *)pdc_vector_iterator_next(tf_obj_id_to_dg_vector_iter); + if (cur_obj_id_to_dg->obj_id == obj_id) { + obj_id_to_dg = cur_obj_id_to_dg; + break; + } + } + pdc_vector_iterator_destroy(tf_obj_id_to_dg_vector_iter); + + // If object has attached graph make sure it is the same as the passed in graph + if (obj_id_to_dg != NULL) { + char *graph_json_filepath = (char *)(obj_id_to_dg->dg->data); + if (strcmp(json_filepath, graph_json_filepath)) { + PGOTO_ERROR(FAIL, "Passed graph filepath %s didn't match stored filepath %s", json_filepath, + graph_json_filepath); + } + } + + // Region mappings for passed in region and object + pdc_tf_region_mapping_t *region_mapping = NULL; + + // If object doesn't have a directed graph create a new one + if (obj_id_to_dg == NULL) { + LOG_DEBUG("Creating directed graph for object\n"); + + pdc_dg_t *dg = PDCtf_dg_json_create_common(json_filepath); + if (dg == NULL) + PGOTO_ERROR(FAIL, "Failed to load JSON\n"); + + // Create new obj id to dg and append to vector + obj_id_to_dg = PDC_malloc(sizeof(pdc_tf_obj_id_to_dg_t)); + pdc_vector_add(tf_obj_id_to_dg_vector_g, obj_id_to_dg); + + obj_id_to_dg->dg = dg; + obj_id_to_dg->obj_id = obj_id; + + // Create a new region mapping vector and mapping entry + obj_id_to_dg->pdc_tf_obj.region_mappings_vector = pdc_vector_create(8, 2.0); + region_mapping = PDC_calloc(1, sizeof(pdc_tf_region_mapping_t)); + pdc_vector_add(obj_id_to_dg->pdc_tf_obj.region_mappings_vector, region_mapping); + } + + // Check if this mapping already exists + if (region_mapping == NULL) { + PDC_VECTOR_ITERATOR *region_mapping_iter = + pdc_vector_iterator_new(obj_id_to_dg->pdc_tf_obj.region_mappings_vector); + while (pdc_vector_iterator_has_next(region_mapping_iter)) { + pdc_tf_region_mapping_t *cur_region_mapping = pdc_vector_iterator_next(region_mapping_iter); + if (cur_region_mapping == NULL) + PGOTO_ERROR(FAIL, "cur_region_mapping was NULL"); + uint64_t *conceptual_offset = cur_region_mapping->conceptual_offset; + if (memcmp(offset, conceptual_offset, ndim * sizeof(uint64_t)) == 0) { + region_mapping = cur_region_mapping; + break; + } + } + pdc_vector_iterator_destroy(region_mapping_iter); + } + + // If this is null we need to append this mapping + if (region_mapping == NULL) { + region_mapping = PDC_calloc(1, sizeof(pdc_tf_region_mapping_t)); + pdc_vector_add(obj_id_to_dg->pdc_tf_obj.region_mappings_vector, region_mapping); + } + + pdc_tf_region_t *conceptual_region = ®ion_mapping->conceptual_region; + uint64_t * conceptual_offset = region_mapping->conceptual_offset; + + PDC_get_var_type_size(pdc_var_type); + + // copy region information into conceptual region + conceptual_region->ndim = ndim; + conceptual_region->pdc_var_type = pdc_var_type; + memcpy(conceptual_offset, offset, ndim * sizeof(uint64_t)); + memcpy(conceptual_region->size, size, ndim * sizeof(uint64_t)); + + LOG_DEBUG("obj_id=%" PRIu64 " ndim=%u\n", obj_id, ndim); + for (int i = 0; i < ndim; i++) { + LOG_DEBUG(" offset[%d]=%" PRIu64 " size[%d]=%" PRIu64 "\n", i, conceptual_offset[i], i, + conceptual_region->size[i]); + } + + // FIXME: need to free these strings later + region_mapping->region_state.cur_state = strdup(cur_state); + region_mapping->region_state.client_state = strdup(client_state); + region_mapping->region_state.store_state = strdup(store_state); + region_mapping->region_state.dg_id = cur_graph; + +done: + FUNC_LEAVE(ret_value); +} + +static double +rolling_avg(double *history) +{ + double avg = 0.0; + for (int i = 0; i < NUM_TF_FUNC_TIMES; i++) + avg += history[i]; + return avg / NUM_TF_FUNC_TIMES; +} + +static double +pure_exec_time(double transform_time, pdc_tf_internal_param internal_params) +{ + double h2d = internal_params.host_to_dev_time >= 0 ? internal_params.host_to_dev_time : 0.0; + double d2h = internal_params.dev_to_host_time >= 0 ? internal_params.dev_to_host_time : 0.0; + return fmax(0.0, transform_time - h2d - d2h); +} + +static void +update_transfer_time(double measured_time, double *history, uint32_t *index, const char *label, + const char *func_name) +{ + if (measured_time < 0) + return; + history[*index] = measured_time; + if (pdc_server_rank_g == 0) + LOG_WARNING("SCHED: updated %s[%d]=%.4f func=%s\n", label, *index, measured_time, func_name); + *index = (*index + 1) % NUM_TF_FUNC_TIMES; +} + +static void +update_exec_time(pdc_tf_func_t *f, double projected_time) +{ + f->exec_avg_time[f->cur_exec_avg_time_index] = projected_time; + if (pdc_server_rank_g == 0) + LOG_WARNING("SCHED: updated %s exec_times[%d]=%.4f new_avg=%.4f func=%s\n", + f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU", f->cur_exec_avg_time_index, projected_time, + rolling_avg(f->exec_avg_time), f->name); + f->cur_exec_avg_time_index = (f->cur_exec_avg_time_index + 1) % NUM_TF_FUNC_TIMES; +} + +static void +update_func_history(pdc_tf_func_t *f, pdc_tf_internal_param internal_params, double projected_time) +{ + update_transfer_time(internal_params.host_to_dev_time, f->host_to_dev_avg_time, + &f->cur_host_to_dev_avg_time_index, "host_to_dev_times", f->name); + update_exec_time(f, projected_time); + update_transfer_time(internal_params.dev_to_host_time, f->dev_to_host_avg_time, + &f->cur_dev_to_host_avg_time_index, "dev_to_host_times", f->name); +} + +static double +expected_exec_time(pdc_tf_func_t *f, double avg_cpu_utilization, double min_avg_gpu_utilization) +{ + double last_avg = rolling_avg(f->exec_avg_time); + return (f->dev == PDC_TF_CPU_DEVICE) ? last_avg / fmax(0.1, 1.0 - avg_cpu_utilization) + : last_avg / fmax(0.1, 1.0 - min_avg_gpu_utilization); +} + +static uint32_t +select_best_edge(pdc_dg_edge_t *edges_out, uint32_t j, uint32_t cur_edges_between_vertices, + double avg_cpu_utilization, double min_avg_gpu_utilization, + int *min_gpu_utilization_device_index_out, bool always_use_gpu, bool *force_selected_out, + double predicted_gpu_ms) +{ + uint32_t best_edge_idx = j; + bool force_selected = false; + + // Static schedule: always pick GPU edge if available + if (always_use_gpu || close_time_g) { + for (uint32_t idx = j; idx < j + cur_edges_between_vertices; idx++) { + pdc_tf_func_t *f = (pdc_tf_func_t *)edges_out[idx].data; + if (f->dev == PDC_TF_GPU_DEVICE) { + best_edge_idx = idx; + *min_gpu_utilization_device_index_out = 0; + force_selected = true; + if (pdc_server_rank_g == 0 && always_use_gpu) + LOG_WARNING("ALWAYS_USE_GPU: selecting edge %u on GPU 0\n", idx); + if (close_time_g && pdc_server_rank_g == 0) + LOG_WARNING("CLOSE_TIME: selecting edge %u on GPU 0\n", idx); + break; + } + } + } + + // Dynamic schedule: select based on expected time and utilization + if (!force_selected) { + double best_expected_time = 1e9; + for (uint32_t idx = j; idx < j + cur_edges_between_vertices; idx++) { + pdc_tf_func_t *f = (pdc_tf_func_t *)edges_out[idx].data; + + double exp_time; + if (f->dev == PDC_TF_GPU_DEVICE && predicted_gpu_ms >= 0.0) { + exp_time = predicted_gpu_ms; + } + else { + double cpu_rolling = rolling_avg(f->exec_avg_time); + exp_time = (cpu_rolling < 0.001) + ? 0.750 + : expected_exec_time(f, avg_cpu_utilization, min_avg_gpu_utilization); + } + if (pdc_server_rank_g == 0) + LOG_WARNING("SCHED: edge %u func=%s dev=%s last_avg_time=%.4f expected_time=%.4f " + "best_so_far=%.4f\n", + idx, f->name, f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU", + rolling_avg(f->exec_avg_time), exp_time, best_expected_time); + + if (exp_time < best_expected_time) { + best_expected_time = exp_time; + best_edge_idx = idx; + } + } + + if (pdc_server_rank_g == 0) { + pdc_tf_func_t *chosen = (pdc_tf_func_t *)edges_out[best_edge_idx].data; + LOG_WARNING("SCHED: chose edge %u func=%s dev=%s expected_time=%.4f\n", best_edge_idx, + chosen->name, chosen->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU", best_expected_time); + } + } + + *force_selected_out = force_selected; + return best_edge_idx; +} + +perr_t +PDCtf_exec_graph(pdc_dg_t *dg, uint64_t flat_conceptual_offset, char *cur_state, char *desired_state, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region, void **input, int is_write, + pdc_tf_sched_mode_t sched_mode) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + PDC_get_var_type_size(input_region.pdc_var_type); + + pdc_tf_state_t tf_input_state = {.name = cur_state}; + pdc_tf_state_t tf_output_state = {.name = desired_state}; + void * input_state = (void *)&tf_input_state; + void * output_state = (void *)&tf_output_state; + + pdc_dg_edge_t *edges_out = NULL; + uint32_t num_edges; + + if (PDCdg_shortest_path(dg, input_state, output_state, &edges_out, &num_edges)) { + memcpy(output_region, &input_region, sizeof(pdc_tf_region_t)); + + for (uint32_t j = 0; j < num_edges;) { + pdc_dg_vertex_id_t v1_id = edges_out[j].v1_id; + pdc_dg_vertex_id_t v2_id = edges_out[j].v2_id; + + uint32_t cur_edges_between_vertices = 0; + uint32_t k = j; + while (k < num_edges && edges_out[k].v1_id == v1_id && edges_out[k].v2_id == v2_id) { + cur_edges_between_vertices++; + k++; + } + + /* hoisted so execute and post-execution blocks can both access */ + int min_gpu_utilization_device_index = 0; + double predicted_total_ms = -1.0; + double avg_cpu_utilization = 0.0; + double min_avg_gpu_utilization = 0.0; + uint32_t best_edge_idx = j; + + /* ── fast path: single edge, skip profiling and scheduling ── */ + if (cur_edges_between_vertices > 1) { + double data_size_mb = (double)PDCtf_get_pdc_region_t_bytes(input_region) / 1e6; + avg_cpu_utilization = pdc_tf_avg_cpu_utilization(); + + /* ── GPU selection ──────────────────────────────────────────── + * DYNAMIC: polynomial predicts total_ms per GPU, pick lowest. + * STATIC: always use GPU 0 (baseline). + * ─────────────────────────────────────────────────────────── */ + if (sched_mode == PDC_TF_SCHED_DYNAMIC) { + pdc_tf_nvml_profiler_update(); + min_gpu_utilization_device_index = pdc_tf_poly_select_gpu(data_size_mb); + + double poly_features[PDC_POLY_N_FEATURES]; + double prev_h2d_ms_log, prev_comp_ms_log, prev_d2h_ms_log, prev_total_ms_log; + pdc_tf_get_device_lag(min_gpu_utilization_device_index, &prev_h2d_ms_log, + &prev_comp_ms_log, &prev_d2h_ms_log, &prev_total_ms_log); + poly_features[0] = data_size_mb; + poly_features[1] = pdc_tf_avg_gpu_utilization(min_gpu_utilization_device_index) * 100.0; + poly_features[2] = prev_h2d_ms_log; + poly_features[3] = prev_d2h_ms_log; + predicted_total_ms = pdc_tf_poly_predict(poly_features); + } + else { + min_gpu_utilization_device_index = 0; + } + + min_avg_gpu_utilization = pdc_tf_avg_gpu_utilization(min_gpu_utilization_device_index); + + /* ── pre-execution logging ────────────────────────────────── */ + if (true || pdc_server_rank_g == 0) { + LOG_WARNING("SCHED: mode=%s avg_cpu_util=%.4f selected_gpu=%d " + "nvml_count=%d close_time_g=%d\n", + sched_mode == PDC_TF_SCHED_DYNAMIC ? "DYNAMIC" : "STATIC", + avg_cpu_utilization, min_gpu_utilization_device_index, + pdc_tf_profiler_nvml_device_count, close_time_g); + + for (int i = 0; i < (int)pdc_tf_profiler_nvml_device_count; i++) { + double ph, pc, pd, pt; + pdc_tf_get_device_lag(i, &ph, &pc, &pd, &pt); + LOG_WARNING("SCHED: GPU[%d] util=%.1f%% power=%.0f mW " + "mem_used=%.0f MB prev_h2d=%.2f ms prev_d2h=%.2f ms " + "prev_total=%.2f ms\n", + i, pdc_tf_avg_gpu_utilization(i) * 100.0, + (double)pdc_tf_avg_gpu_power_mw(i), + (double)(pdc_tf_avg_gpu_mem_used(i) / (1024UL * 1024UL)), ph, pd, pt); + } + + if (sched_mode == PDC_TF_SCHED_DYNAMIC) + LOG_WARNING("SCHED: data_size=%.2f MB predicted_total=%.2f ms " + "selected_gpu=%d\n", + data_size_mb, predicted_total_ms, min_gpu_utilization_device_index); + else + LOG_WARNING("SCHED: data_size=%.2f MB static_gpu=0\n", data_size_mb); + } + + /* ── edge selection ───────────────────────────────────────── */ + bool force_selected = false; + best_edge_idx = select_best_edge( + edges_out, j, cur_edges_between_vertices, avg_cpu_utilization, min_avg_gpu_utilization, + &min_gpu_utilization_device_index, &force_selected, false, predicted_total_ms / 1000.0); + } + + /* ── execute transformation ───────────────────────────────── */ + pdc_dg_edge_t e = edges_out[best_edge_idx]; + pdc_tf_func_t *f = (pdc_tf_func_t *)(e.data); + + pdc_tf_internal_param internal_params = {0}; + internal_params.dg = dg; + internal_params.flat_conceptual_offset = flat_conceptual_offset; + internal_params.host_to_dev_time = 0; + internal_params.dev_to_host_time = 0; + + if (f->dev == PDC_TF_GPU_DEVICE) { + int gpu_idx = close_time_g ? 0 : min_gpu_utilization_device_index; + LOG_WARNING("SCHED: setting CUDA device to %d for edge %u func=%s\n", gpu_idx, best_edge_idx, + f->name); + cudaError_t err = cudaSetDevice(gpu_idx); + if (err != cudaSuccess) + PGOTO_ERROR(FAIL, "Failed to set device %d\n", gpu_idx); + } + + LOG_WARNING("SCHED: executing edge %u func=%s dev=%s\n", best_edge_idx, f->name, + f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU"); + + void * prev_input = *input; + struct timespec start_time, end_time; + clock_gettime(CLOCK_MONOTONIC, &start_time); + + memcpy(&input_region, output_region, sizeof(pdc_tf_region_t)); + if (!f->c_func(&internal_params, f->params_str, input, input_region, output_region)) + PGOTO_ERROR(FAIL, "Error running transformation %s", f->name); + + clock_gettime(CLOCK_MONOTONIC, &end_time); + double transform_time = + (end_time.tv_sec - start_time.tv_sec) + (end_time.tv_nsec - start_time.tv_nsec) / 1e9; + double transform_time_ms = transform_time * 1000.0; + + /* ── post-execution: only for multi-edge paths ────────────── */ + if (cur_edges_between_vertices > 1) { + double exec_only_time = pure_exec_time(transform_time, internal_params); + double projected_time = (f->dev == PDC_TF_CPU_DEVICE) + ? exec_only_time * (1.0 - fmax(avg_cpu_utilization, 0.1)) + : exec_only_time * (1.0 - fmax(min_avg_gpu_utilization, 0.1)); + + double h2d_ms = + internal_params.host_to_dev_time >= 0 ? internal_params.host_to_dev_time * 1000.0 : 0.0; + double d2h_ms = + internal_params.dev_to_host_time >= 0 ? internal_params.dev_to_host_time * 1000.0 : 0.0; + double comp_ms = exec_only_time * 1000.0; + + if (true || pdc_server_rank_g == 0) { + LOG_WARNING("TIMING: mode=%s func=%s dev=%s gpu=%d " + "h2d=%.4f ms comp=%.4f ms d2h=%.4f ms " + "total=%.4f ms exec_only=%.4f ms projected=%.4f ms", + sched_mode == PDC_TF_SCHED_DYNAMIC ? "DYNAMIC" : "STATIC", f->name, + f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU", min_gpu_utilization_device_index, + h2d_ms, comp_ms, d2h_ms, transform_time_ms, exec_only_time * 1000.0, + projected_time * 1000.0); + + if (sched_mode == PDC_TF_SCHED_DYNAMIC && f->dev == PDC_TF_GPU_DEVICE) + LOG_WARNING("TIMING: predicted=%.4f ms actual=%.4f ms error=%.4f ms\n", + predicted_total_ms, transform_time_ms, + transform_time_ms - predicted_total_ms); + else + LOG_WARNING("\n"); + } + + update_func_history(f, internal_params, projected_time); + + if (sched_mode == PDC_TF_SCHED_DYNAMIC) { + pdc_tf_poly_update(min_gpu_utilization_device_index, + internal_params.host_to_dev_time >= 0 ? h2d_ms : -1.0, comp_ms, + internal_params.dev_to_host_time >= 0 ? d2h_ms : -1.0, + transform_time_ms); + } + } + + if (!(is_write && j == 0) && prev_input != *input) + prev_input = PDC_free(prev_input); + + if (j + cur_edges_between_vertices != num_edges) + memcpy(&input_region, output_region, sizeof(pdc_tf_region_t)); + + j += cur_edges_between_vertices; + } + + LOG_WARNING("SCHED: done running transformations\n"); + } + else { + LOG_ERROR("JSON filepath %s\n", (char *)dg->data); + LOG_ERROR("Current state %s, desired state %s\n", cur_state, desired_state); + PGOTO_ERROR(FAIL, "No path to desired state"); + } + +done: + if (edges_out != NULL) + edges_out = PDC_free(edges_out); + + FUNC_LEAVE(ret_value); +} + +perr_t +PDCtf_exec_graph_backup(pdc_dg_t *dg, uint64_t flat_conceptual_offset, char *cur_state, char *desired_state, + pdc_tf_region_t input_region, pdc_tf_region_t *output_region, void **input, + int is_write, pdc_tf_sched_mode_t sched_mode) +{ + FUNC_ENTER(NULL); + + perr_t ret_value = SUCCEED; + + PDC_get_var_type_size(input_region.pdc_var_type); + + pdc_tf_state_t tf_input_state = {.name = cur_state}; + pdc_tf_state_t tf_output_state = {.name = desired_state}; + void * input_state = (void *)&tf_input_state; + void * output_state = (void *)&tf_output_state; + + pdc_dg_edge_t *edges_out = NULL; + uint32_t num_edges; + + const char *value = getenv("USE_GPU"); + bool always_use_gpu = (value != NULL); + + if (PDCdg_shortest_path(dg, input_state, output_state, &edges_out, &num_edges)) { + memcpy(output_region, &input_region, sizeof(pdc_tf_region_t)); + + for (uint32_t j = 0; j < num_edges;) { + pdc_dg_vertex_id_t v1_id = edges_out[j].v1_id; + pdc_dg_vertex_id_t v2_id = edges_out[j].v2_id; + + uint32_t cur_edges_between_vertices = 0; + uint32_t k = j; + while (k < num_edges && edges_out[k].v1_id == v1_id && edges_out[k].v2_id == v2_id) { + cur_edges_between_vertices++; + k++; + } + + /* ── device state ─────────────────────────────────────────── */ + double data_size_mb = (double)PDCtf_get_pdc_region_t_bytes(input_region) / 1e6; + double avg_cpu_utilization = pdc_tf_avg_cpu_utilization(); + + /* ── GPU selection ──────────────────────────────────────────── + * DYNAMIC: polynomial predicts total_ms per GPU, pick lowest. + * STATIC: always use GPU 0 (baseline). + * ─────────────────────────────────────────────────────────── */ + int min_gpu_utilization_device_index; + double predicted_total_ms = -1.0; + + if (sched_mode == PDC_TF_SCHED_DYNAMIC) { + pdc_tf_nvml_profiler_update(); + min_gpu_utilization_device_index = pdc_tf_poly_select_gpu(data_size_mb); + + /* build feature vector for logging predicted vs actual */ + double poly_features[PDC_POLY_N_FEATURES]; + double prev_h2d_ms_log, prev_comp_ms_log, prev_d2h_ms_log, prev_total_ms_log; + pdc_tf_get_device_lag(min_gpu_utilization_device_index, &prev_h2d_ms_log, &prev_comp_ms_log, + &prev_d2h_ms_log, &prev_total_ms_log); + poly_features[0] = data_size_mb; + poly_features[1] = pdc_tf_avg_gpu_utilization(min_gpu_utilization_device_index) * 100.0; + poly_features[2] = prev_h2d_ms_log; + poly_features[3] = prev_d2h_ms_log; + predicted_total_ms = pdc_tf_poly_predict(poly_features); + } + else { + /* PDC_TF_SCHED_STATIC: always GPU 0 */ + min_gpu_utilization_device_index = 0; + } + + double min_avg_gpu_utilization = pdc_tf_avg_gpu_utilization(min_gpu_utilization_device_index); + + /* ── pre-execution logging ────────────────────────────────── */ + if (true || pdc_server_rank_g == 0) { + LOG_WARNING("SCHED: mode=%s avg_cpu_util=%.4f selected_gpu=%d " + "nvml_count=%d always_use_gpu=%d close_time_g=%d\n", + sched_mode == PDC_TF_SCHED_DYNAMIC ? "DYNAMIC" : "STATIC", avg_cpu_utilization, + min_gpu_utilization_device_index, pdc_tf_profiler_nvml_device_count, + always_use_gpu, close_time_g); + + /* per-GPU state used by polynomial */ + for (int i = 0; i < (int)pdc_tf_profiler_nvml_device_count; i++) { + double ph, pc, pd, pt; + pdc_tf_get_device_lag(i, &ph, &pc, &pd, &pt); + LOG_WARNING("SCHED: GPU[%d] util=%.1f%% power=%.0f mW " + "mem_used=%.0f MB prev_h2d=%.2f ms prev_d2h=%.2f ms " + "prev_total=%.2f ms\n", + i, pdc_tf_avg_gpu_utilization(i) * 100.0, (double)pdc_tf_avg_gpu_power_mw(i), + (double)(pdc_tf_avg_gpu_mem_used(i) / (1024UL * 1024UL)), ph, pd, pt); + } + + if (sched_mode == PDC_TF_SCHED_DYNAMIC) + LOG_WARNING("SCHED: data_size=%.2f MB predicted_total=%.2f ms " + "selected_gpu=%d\n", + data_size_mb, predicted_total_ms, min_gpu_utilization_device_index); + else + LOG_WARNING("SCHED: data_size=%.2f MB static_gpu=0\n", data_size_mb); + } + + /* ── edge selection ───────────────────────────────────────── */ + uint32_t best_edge_idx = select_best_edge( + edges_out, j, cur_edges_between_vertices, avg_cpu_utilization, min_avg_gpu_utilization, + &min_gpu_utilization_device_index, always_use_gpu, false, predicted_total_ms / 1000.0); + + /* ── execute transformation ───────────────────────────────── */ + pdc_dg_edge_t e = edges_out[best_edge_idx]; + pdc_tf_func_t *f = (pdc_tf_func_t *)(e.data); + + pdc_tf_internal_param internal_params = {0}; + internal_params.dg = dg; + internal_params.flat_conceptual_offset = flat_conceptual_offset; + internal_params.host_to_dev_time = 0; + internal_params.dev_to_host_time = 0; + + if (f->dev == PDC_TF_GPU_DEVICE) { + LOG_WARNING("SCHED: setting CUDA device to %d for edge %u func=%s\n", + (always_use_gpu || close_time_g) ? 0 : min_gpu_utilization_device_index, + best_edge_idx, f->name); + cudaError_t err = + cudaSetDevice((always_use_gpu || close_time_g) ? 0 : min_gpu_utilization_device_index); + if (err != cudaSuccess) + PGOTO_ERROR(FAIL, "Failed to set device %d\n", + (always_use_gpu || close_time_g) ? 0 : min_gpu_utilization_device_index); + } + + LOG_WARNING("SCHED: executing edge %u func=%s dev=%s\n", best_edge_idx, f->name, + f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU"); + + void * prev_input = *input; + struct timespec start_time, end_time; + clock_gettime(CLOCK_MONOTONIC, &start_time); + + memcpy(&input_region, output_region, sizeof(pdc_tf_region_t)); + if (!f->c_func(&internal_params, f->params_str, input, input_region, output_region)) + PGOTO_ERROR(FAIL, "Error running transformation %s", f->name); + + clock_gettime(CLOCK_MONOTONIC, &end_time); + double transform_time = + (end_time.tv_sec - start_time.tv_sec) + (end_time.tv_nsec - start_time.tv_nsec) / 1e9; + double transform_time_ms = transform_time * 1000.0; + + double exec_only_time = pure_exec_time(transform_time, internal_params); + double projected_time = (f->dev == PDC_TF_CPU_DEVICE) + ? exec_only_time * (1.0 - fmax(avg_cpu_utilization, 0.1)) + : exec_only_time * (1.0 - fmax(min_avg_gpu_utilization, 0.1)); + + double h2d_ms = + internal_params.host_to_dev_time >= 0 ? internal_params.host_to_dev_time * 1000.0 : 0.0; + double d2h_ms = + internal_params.dev_to_host_time >= 0 ? internal_params.dev_to_host_time * 1000.0 : 0.0; + double comp_ms = exec_only_time * 1000.0; + + /* ── post-execution logging ─────────────────────────────────── + * Log all timing components plus predicted vs actual for + * dynamic mode so we can evaluate prediction accuracy offline. + * ─────────────────────────────────────────────────────────── */ + if (true || pdc_server_rank_g == 0) { + LOG_WARNING("TIMING: mode=%s func=%s dev=%s gpu=%d " + "h2d=%.4f ms comp=%.4f ms d2h=%.4f ms " + "total=%.4f ms exec_only=%.4f ms projected=%.4f ms", + sched_mode == PDC_TF_SCHED_DYNAMIC ? "DYNAMIC" : "STATIC", f->name, + f->dev == PDC_TF_CPU_DEVICE ? "CPU" : "GPU", min_gpu_utilization_device_index, + h2d_ms, comp_ms, d2h_ms, transform_time_ms, exec_only_time * 1000.0, + projected_time * 1000.0); + + if (sched_mode == PDC_TF_SCHED_DYNAMIC && f->dev == PDC_TF_GPU_DEVICE) + LOG_WARNING("TIMING: predicted=%.4f ms actual=%.4f ms error=%.4f ms\n", + predicted_total_ms, transform_time_ms, + transform_time_ms - predicted_total_ms); + else + LOG_WARNING("\n"); + } + + update_func_history(f, internal_params, projected_time); + + /* ── update lag features ────────────────────────────────────── + * Record actual timings for this GPU so they become lag + * features for the next scheduling decision on this device. + * Only updated in DYNAMIC mode — STATIC has no model to feed. + * ─────────────────────────────────────────────────────────── */ + if (sched_mode == PDC_TF_SCHED_DYNAMIC) { + pdc_tf_poly_update(min_gpu_utilization_device_index, + internal_params.host_to_dev_time >= 0 ? h2d_ms : -1.0, comp_ms, + internal_params.dev_to_host_time >= 0 ? d2h_ms : -1.0, transform_time_ms); + } + + if (!(is_write && j == 0) && prev_input != *input) + prev_input = PDC_free(prev_input); + + if (j + cur_edges_between_vertices != num_edges) + memcpy(&input_region, output_region, sizeof(pdc_tf_region_t)); + + j += cur_edges_between_vertices; + } + + LOG_WARNING("SCHED: done running transformations\n"); + } + else { + LOG_ERROR("JSON filepath %s\n", (char *)dg->data); + LOG_ERROR("Current state %s, desired state %s\n", cur_state, desired_state); + PGOTO_ERROR(FAIL, "No path to desired state"); + } + +done: + if (edges_out != NULL) + edges_out = PDC_free(edges_out); + + FUNC_LEAVE(ret_value); +} + +#endif // IS_PDC_SERVER diff --git a/src/server/transform/pdc_tf_user.c b/src/server/transform/pdc_tf_user.c new file mode 100644 index 000000000..c38c304f1 --- /dev/null +++ b/src/server/transform/pdc_tf_user.c @@ -0,0 +1,212 @@ +/** + * NOTICE: This file is included in a custom transformation libaries. + * It should not include any headers that are not available to the custom transformation library. + * It should also not include any headers that define symbols that + * may conflict with symbols in the custom transformation library. + */ + +#include "pdc_tf_user.h" +#include +#include +#include + +bool +PDCtf_set_func_param(pdc_dg_t *dg, char *func_name, pdc_tf_dev_t dev, uint64_t flat_conceptual_offset, + void *params, uint64_t params_size) +{ + + bool ret_value = true; + PDC_VECTOR_ITERATOR *dg_params_iter = NULL; + + // Find edge with name + for (int i = 0; i < dg->edge_count; i++) { + pdc_dg_edge_t *edge = dg->edges[i]; + pdc_tf_func_t *tf_func = edge->data; + + if (!strcmp(func_name, tf_func->name)) { + // Create vector if vector is NULL + if (tf_func->pdc_tf_dg_params_vector == NULL) + tf_func->pdc_tf_dg_params_vector = pdc_vector_create(8, 2.0); + + // Locate the params by conceptual offset + pdc_tf_dg_params_t *pdc_dg_params = NULL; + dg_params_iter = pdc_vector_iterator_new(tf_func->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_dg_params = pdc_vector_iterator_next(dg_params_iter); + if (pdc_dg_params->flat_conceptual_offset == flat_conceptual_offset) { + pdc_dg_params->params = params; + pdc_dg_params->params_size = params_size; + goto done; + } + } + + // Append new entry + pdc_dg_params = calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(tf_func->pdc_tf_dg_params_vector, pdc_dg_params); + + pdc_dg_params->flat_conceptual_offset = flat_conceptual_offset; + pdc_dg_params->params = params; + pdc_dg_params->params_size = params_size; + goto done; + } + } + + printf("Edge %s not found\n", func_name); + +done: + if (dg_params_iter != NULL) + pdc_vector_iterator_destroy(dg_params_iter); + return ret_value; +} + +bool +PDCtf_get_func_param(pdc_dg_t *dg, char *func_name, pdc_tf_dev_t dev, uint64_t flat_conceptual_offset, + void **params, uint64_t *params_size) +{ + bool ret_value = true; + PDC_VECTOR_ITERATOR *dg_params_iter = NULL; + + // Find edge with name + for (int i = 0; i < dg->edge_count; i++) { + pdc_dg_edge_t *edge = dg->edges[i]; + pdc_tf_func_t *tf_func = edge->data; + if (!strcmp(func_name, tf_func->name) && tf_func->dev == dev) { + // Check if vector is NULL + if (tf_func->pdc_tf_dg_params_vector == NULL) { + printf("tf_func->pdc_tf_dg_params_vector was NULL\n"); + return false; + goto done; + } + + // Locate the params by conceptual offset + dg_params_iter = pdc_vector_iterator_new(tf_func->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_tf_dg_params_t *pdc_dg_params = pdc_vector_iterator_next(dg_params_iter); + if (pdc_dg_params->flat_conceptual_offset == flat_conceptual_offset) { + *params = pdc_dg_params->params; + *params_size = pdc_dg_params->params_size; + goto done; + } + } + printf("Failed to locate params in func_name %s by flat conceptual offset %lu\n", func_name, + flat_conceptual_offset); + return false; + goto done; + } + } + + printf("Edge %s not found\n", func_name); + ret_value = false; + +done: + if (dg_params_iter != NULL) + pdc_vector_iterator_destroy(dg_params_iter); + return ret_value; +} + +bool +PDCtf_set_state_param(pdc_dg_t *dg, char *state_name, uint64_t flat_conceptual_offset, void *params, + uint64_t params_size) +{ + bool ret_value = true; + PDC_VECTOR_ITERATOR *dg_params_iter = NULL; + + // Get state from graph + pdc_tf_state_t query_stat; + query_stat.name = state_name; + + pdc_dg_vertex_id_t vert = PDCdg_vertex_exists(dg, &query_stat); + if (vert == PDC_DG_INVALID_VERTEX) { + printf("Failed to find state in PDCtf_set_state_param\n"); + ret_value = false; + goto done; + } + + // Get the tf state + pdc_tf_state_t *tf_state = (pdc_tf_state_t *)dg->vertices[vert]->data; + if (tf_state == NULL) { + printf("Vertex data was NULL\n"); + ret_value = false; + goto done; + } + + // Create vector if vector is NULL + if (tf_state->pdc_tf_dg_params_vector == NULL) + tf_state->pdc_tf_dg_params_vector = pdc_vector_create(8, 2.0); + + // Locate the params by conceptual offset + pdc_tf_dg_params_t *pdc_dg_params = NULL; + dg_params_iter = pdc_vector_iterator_new(tf_state->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_dg_params = pdc_vector_iterator_next(dg_params_iter); + if (pdc_dg_params->flat_conceptual_offset == flat_conceptual_offset) { + pdc_dg_params->params = params; + pdc_dg_params->params_size = params_size; + goto done; + } + } + + pdc_dg_params = calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(tf_state->pdc_tf_dg_params_vector, pdc_dg_params); + + pdc_dg_params->params = params; + pdc_dg_params->params_size = params_size; + pdc_dg_params->flat_conceptual_offset = flat_conceptual_offset; + +done: + if (dg_params_iter != NULL) + pdc_vector_iterator_destroy(dg_params_iter); + return ret_value; +} + +bool +PDCtf_get_state_param(pdc_dg_t *dg, char *state_name, uint64_t flat_conceptual_offset, void **params, + uint64_t *params_size) +{ + bool ret_value = true; + PDC_VECTOR_ITERATOR *dg_params_iter = NULL; + + // Get state from graph + pdc_tf_state_t query_stat = {.name = state_name}; + + pdc_dg_vertex_id_t vert = PDCdg_vertex_exists(dg, &query_stat); + if (vert == PDC_DG_INVALID_VERTEX) { + printf("Failed to find state in PDCtf_get_state_param\n"); + ret_value = false; + goto done; + } + + // Get the tf state + pdc_tf_state_t *tf_state = (pdc_tf_state_t *)dg->vertices[vert]->data; + if (tf_state == NULL) { + printf("Vertex data was NULL\n"); + ret_value = false; + goto done; + } + + // Check if vector is NULL + if (tf_state->pdc_tf_dg_params_vector == NULL) { + printf("tf_state->pdc_tf_dg_params_vector was NULL\n"); + ret_value = false; + goto done; + } + + // Locate the params by conceptual offset + dg_params_iter = pdc_vector_iterator_new(tf_state->pdc_tf_dg_params_vector); + while (pdc_vector_iterator_has_next(dg_params_iter)) { + pdc_tf_dg_params_t *pdc_dg_params = pdc_vector_iterator_next(dg_params_iter); + if (pdc_dg_params->flat_conceptual_offset == flat_conceptual_offset) { + *params = pdc_dg_params->params; + *params_size = pdc_dg_params->params_size; + goto done; + } + } + + printf("Failed to locate params in state_name %s by flat conceptual offset %lu\n", state_name, + flat_conceptual_offset); + +done: + if (dg_params_iter != NULL) + pdc_vector_iterator_destroy(dg_params_iter); + return ret_value; +} diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6f0482fdc..db24798c8 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -102,32 +102,82 @@ set(PROGRAMS region/region_transfer_set_dims_3D query/query_data misc/producer_waitall + + transformation/compression_transform + transformation/compression_transform_2D + transformation/compression_transform_3D + transformation/compression_transform_gpu + transformation/attach_obj_compression_transform + transformation/multi_region_transform + transformation/compression_encryption_transform + transformation/compression_encryption_transform_2D + transformation/compression_encryption_transform_3D + transformation/all_compression_transform +) + +# Use C compiler for C files +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) # still needed for linking C++/CUDA + +# Path to your CUDA library +if(DEFINED ENV{CUDA_ROOT_DIR} AND NOT "$ENV{CUDA_ROOT_DIR}" STREQUAL "") + set(CUDA_ROOT $ENV{CUDA_ROOT_DIR}) +endif() + +# Path to your existing CUDA library +find_package(MPI REQUIRED) +set(PI_GPU_LIB_PATH /pscratch/sd/n/nlewi26/src/work_space/source/pdc/libpi_gpu.so) +set(PI_GPU_INCLUDE_PATH /pscratch/sd/n/nlewi26/src/work_space/source/pdc) + +# Create an imported library target for your CUDA library +add_library(pi_gpu SHARED IMPORTED) +set_target_properties(pi_gpu PROPERTIES + IMPORTED_LOCATION ${PI_GPU_LIB_PATH} + INTERFACE_INCLUDE_DIRECTORIES ${PI_GPU_INCLUDE_PATH} ) -# TODO: Check if import_vpic.c is needed. If yes, we have to add the following : -# if (HDF5_FOUND) -# set(TEST_EXT_LIB ${HDF5_LIBRARIES} ${TEST_EXT_LIB}) -# set(TEST_EXT_INCLUDE_DIRS ${HDF5_INCLUDE_DIRS} ${TEST_EXT_INCLUDE_DIRS}) -# else () -# message(FATAL_ERROR "Could not find HDF5, please make sure that HDF5 has been compiled with shared libraries enabled.") -# endif() +# Link CUDA runtime from your CUDA_ROOT +set(CUDA_RUNTIME_LIB ${CUDA_ROOT}/lib64/libcudart.so) +target_link_libraries(pi_gpu INTERFACE ${CUDA_RUNTIME_LIB}) +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +# Iterate through your programs foreach(program ${PROGRAMS}) - get_filename_component(program_name ${program} NAME) - get_filename_component(program_dir ${program} DIRECTORY) + get_filename_component(program_name ${program} NAME) + get_filename_component(program_dir ${program} DIRECTORY) + set(SOURCE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${program}.c") - set(SOURCE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${program}.c") + if(EXISTS ${SOURCE_FILE}) + add_executable(${program_name} ${SOURCE_FILE}) + + # Use C linker for pure C source files + set_target_properties(${program_name} PROPERTIES LINKER_LANGUAGE C) + + # Include directories + target_include_directories(${program_name} PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/tests/include + ${PI_GPU_INCLUDE_PATH} + ${MPI_C_INCLUDE_PATH} + ) - if(EXISTS ${SOURCE_FILE}) - add_executable(${program_name} ${SOURCE_FILE}) - target_include_directories(${program_name} PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/tests/include - ) - target_link_libraries(${program_name} pdc pdc_commons ${TEST_EXT_LIB}) - else() - message(WARNING "Source file ${SOURCE_FILE} not found, skipping ${program_name}.") - endif() + # Link libraries + target_link_libraries(${program_name} + pi_gpu + pdc + pdc_commons + ${TEST_EXT_LIB} + ${MERCURY_LIBRARY} + ${PDC_COMMONS_LIBRARIES} + -lm -ldl + ${PDC_EXT_LIB_DEPENDENCIES} + ${ZFP_LIB} + ${MPI_C_LIBRARIES} + ) + else() + message(WARNING "Source file ${SOURCE_FILE} not found, skipping ${program_name}.") + endif() endforeach() if(BUILD_MPI_TESTING) @@ -160,6 +210,13 @@ endif() target_include_directories(${program_name} PRIVATE ${TEST_EXT_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/src/tests/include) + target_link_libraries(${program_name} + ${MERCURY_LIBRARY} + ${PDC_COMMONS_LIBRARIES} + -lm -ldl + ${PDC_EXT_LIB_DEPENDENCIES} + ${ZFP_LIB} + ) else() message(WARNING "Source file ${SOURCE_FILE} not found, skipping ${program_name}.") endif() @@ -176,7 +233,6 @@ if(UUID_FOUND) dart/dart_func_test dart/dart_test ) - set(TEST_EXT_LIB ${TEST_EXT_LIB} ${UUID_LIBRARIES}) set(TEST_EXT_INCLUDE_DIRS ${TEST_EXT_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/helper/include) @@ -203,6 +259,13 @@ if(UUID_FOUND) add_executable(${program_name} ${SOURCE_FILE} ${EXTRA_SRC_FILE}) target_link_libraries(${program_name} pdc pdc_commons ${TEST_EXT_LIB}) target_include_directories(${program_name} PRIVATE ${TEST_EXT_INCLUDE_DIRS}) + target_link_libraries(${program_name} + ${MERCURY_LIBRARY} + ${PDC_COMMONS_LIBRARIES} + -lm -ldl + ${PDC_EXT_LIB_DEPENDENCIES} + ${ZFP_LIB} + ) else() message(WARNING "Source file ${SOURCE_FILE} not found, skipping ${program_name}.") endif() @@ -225,27 +288,6 @@ foreach(script ${SCRIPTS}) ) endforeach(script) - -# ******************************************* -# Create a transform library which contains: -# 1. compression/depression functions -# 2. datatype conversion -# ******************************************* -OPTION(GEN_TRANSFORM_LIB "Create a transform library" OFF) -if(GEN_TRANSFORM_LIB) - FIND_LIBRARY(BLOSC_LIBRARY blosc $ENV{HOME}/Sandbox/c-blosc/build/blosc) - set(PDC_TRANSFORMS_SRCS transforms/pdc_transforms_lib.c) - add_library(pdctransforms ${PDC_TRANSFORMS_SRCS}) - target_link_libraries(pdctransforms pdc ${BLOSC_LIBRARY}) -endif() - -# ******************************************* -# Create a transform library for testing -# ******************************************* -set(PDC_ANALYSIS_SRCS analysis/pdc_analysis_lib.c) -add_library(pdcanalysis ${PDC_ANALYSIS_SRCS}) -target_link_libraries(pdcanalysis pdc) - add_test(NAME pdc_init WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./pdc_init ) add_test(NAME dup_prop WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./dup_prop ) add_test(NAME open_cont WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./open_cont ) @@ -282,6 +324,26 @@ add_test(NAME region_transfer_overlap_2D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPU add_test(NAME region_transfer_overlap_3D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_overlap_3D ) add_test(NAME region_transfer_merge WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_merge ) add_test(NAME region_transfer_write_read WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_write_read ) + +# Transformations region transfer compression +add_test(NAME compression_transform WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_transform) +add_test(NAME compression_transform_2D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_transform_2D) +add_test(NAME compression_transform_3D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_transform_3D) +# Transformations region transfer compression +if(CUDA_ROOT) +add_test(NAME compression_transform_gpu WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_transform_gpu) +endif() +# Transformations attached to object region transfer compression +add_test(NAME attach_obj_compression_transform WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./attach_obj_compression_transform) +# Transformations multi-region transfer +add_test(NAME multi_region_transform WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./multi_region_transform) +# Transformations region transfer compression encryption +add_test(NAME compression_encryption_transform WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_encryption_transform) +add_test(NAME compression_encryption_transform_2D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_encryption_transform_2D) +add_test(NAME compression_encryption_transform_3D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./compression_encryption_transform_3D) +# Transformations region transfer compression all +add_test(NAME all_compression_transform WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./all_compression_transform 0 0) + add_test(NAME region_transfer_all WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_all ) add_test(NAME region_transfer_all_2D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_all_2D ) add_test(NAME region_transfer_all_3D WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND run_test.sh ./region_transfer_all_3D ) @@ -390,6 +452,25 @@ set_tests_properties(read_obj_int16 PROPERTIES LABELS serial) set_tests_properties(read_obj_int8 PROPERTIES LABELS serial) set_tests_properties(vpicio_bdcats_transfer_request PROPERTIES LABELS serial) +# Transformations region transfer compression +set_tests_properties(compression_transform PROPERTIES LABELS transform) +set_tests_properties(compression_transform_2D PROPERTIES LABELS transform) +set_tests_properties(compression_transform_3D PROPERTIES LABELS transform) +# Transformations region transfer compression GPU +if(CUDA_ROOT) +set_tests_properties(compression_transform PROPERTIES LABELS transform_gpu) +endif() +# Transformations attached to object region transfer compression +set_tests_properties(attach_obj_compression_transform PROPERTIES LABELS transform) +# Transformations region transfer multiregion +set_tests_properties(multi_region_transform PROPERTIES LABELS transform) +# Transformations region transfer compression encryption +set_tests_properties(compression_encryption_transform PROPERTIES LABELS transform) +set_tests_properties(compression_encryption_transform_2D PROPERTIES LABELS transform) +set_tests_properties(compression_encryption_transform_3D PROPERTIES LABELS transform) +# Transformations region transfer compression +set_tests_properties(all_compression_transform PROPERTIES LABELS transform) + if(BUILD_MPI_TESTING) add_test(NAME read_obj_shared_int WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./read_obj_shared ${MPI_RUN_CMD} 4 6 o 1 int) add_test(NAME read_obj_shared_float WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND mpi_test.sh ./read_obj_shared ${MPI_RUN_CMD} 4 6 o 1 float) diff --git a/src/tests/analysis/pdc_analysis_lib.c b/src/tests/analysis/pdc_analysis_lib.c deleted file mode 100644 index df877ec06..000000000 --- a/src/tests/analysis/pdc_analysis_lib.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "pdc.h" - -/* Sum the elements of a row to produce a single output */ - -int -demo_sum(pdcid_t iterIn, pdcid_t iterOut) -{ - int i, k, total = 0; - int *dataIn = NULL; - int *dataOut = NULL; - int blockLengthOut, blockLengthIn; - LOG_INFO("Entered: \n----------------\n"); - - if ((blockLengthOut = PDCobj_data_getNextBlock(iterOut, (void **)&dataOut, NULL)) > 0) { - while ((blockLengthIn = PDCobj_data_getNextBlock(iterIn, (void **)&dataIn, NULL)) > 0) { - LOG_INFO("Summing %d elements\n", blockLengthIn); - for (i = 0, k = 0; i < blockLengthIn; i++) { - LOG_INFO("\t%d\n", dataIn[i]); - total += dataIn[i]; - } - LOG_INFO("\nSum = %d\n", total); - dataOut[k++] = total; - total = 0; - } - } - LOG_INFO("Leaving: \n----------------\n"); - return 0; -} - -/* int check_mpi_access(pdcid_t iterIn , pdcid_t iterOut) */ -int -check_mpi_access() -{ - int initialized = 0; -#ifdef ENABLE_MPI - MPI_Initialized(&initialized); -#endif - if (initialized) { - int rank; - int size; - -#ifdef ENABLE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - LOG_INFO("MPI rank is %d of %d\n", rank, size); -#endif - } - - return 0; -} diff --git a/src/tests/deprecated/bdcats_v2.c b/src/tests/deprecated/bdcats_v2.c deleted file mode 100644 index 02e80c709..000000000 --- a/src/tests/deprecated/bdcats_v2.c +++ /dev/null @@ -1,467 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "pdc.h" - -#define NPARTICLES 8388608 - -double -uniform_random_number() -{ - return (((double)rand()) / ((double)(RAND_MAX))); -} - -void -print_usage() -{ - LOG_JUST_PRINT("Usage: srun -n ./vpicio #particles\n"); -} - -int -main(int argc, char **argv) -{ - int rank = 0, size = 1; - pdcid_t pdc_id, cont_id; - pdcid_t obj_xx, obj_yy, obj_zz, obj_pxx, obj_pyy, obj_pzz, obj_id11, obj_id22; - pdcid_t region_x, region_y, region_z, region_px, region_py, region_pz, region_id1, region_id2; - pdcid_t region_xx, region_yy, region_zz, region_pxx, region_pyy, region_pzz, region_id11, region_id22; - perr_t ret; - struct timeval ht_total_start; - struct timeval ht_total_end; - long long ht_total_elapsed; - double ht_total_sec; - float * x, *y, *z; - float * px, *py, *pz; - int * id1, *id2; - uint64_t numparticles; - int ndim = 1; - uint64_t * offset; - uint64_t * offset_remote; - uint64_t * mysize; - -#ifdef ENABLE_MPI - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); -#endif - - numparticles = NPARTICLES; - - x = (float *)malloc(numparticles * sizeof(float)); - y = (float *)malloc(numparticles * sizeof(float)); - z = (float *)malloc(numparticles * sizeof(float)); - - px = (float *)malloc(numparticles * sizeof(float)); - py = (float *)malloc(numparticles * sizeof(float)); - pz = (float *)malloc(numparticles * sizeof(float)); - - id1 = (int *)malloc(numparticles * sizeof(int)); - id2 = (int *)malloc(numparticles * sizeof(int)); - - // create a pdc - pdc_id = PDCinit("pdc"); - - // open a container - cont_id = PDCcont_open("c1", pdc_id); - if (cont_id == 0) - LOG_ERROR("Failed to create container"); - - // open objects - obj_xx = PDCobj_open("obj-var-xx", pdc_id); - if (obj_xx == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-xx"); - exit(-1); - } - obj_yy = PDCobj_open("obj-var-yy", pdc_id); - if (obj_yy == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-yy"); - exit(-1); - } - obj_zz = PDCobj_open("obj-var-zz", pdc_id); - if (obj_zz == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-zz"); - exit(-1); - } - obj_pxx = PDCobj_open("obj-var-pxx", pdc_id); - if (obj_pxx == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-pxx"); - exit(-1); - } - obj_pyy = PDCobj_open("obj-var-pyy", pdc_id); - if (obj_pyy == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-pyy"); - exit(-1); - } - obj_pzz = PDCobj_open("obj-var-pzz", pdc_id); - if (obj_pzz == 0) { - LOG_ERROR("Error when open object %s\n", "obj-var-pzz"); - exit(-1); - } - obj_id11 = PDCobj_open("id11", pdc_id); - if (obj_id11 == 0) { - LOG_ERROR("Error when open object %s\n", "id11"); - exit(-1); - } - obj_id22 = PDCobj_open("id22", pdc_id); - if (obj_id22 == 0) { - LOG_ERROR("Error when open object %s\n", "id22"); - exit(-1); - } - - offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset_remote = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - mysize = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset[0] = 0; - offset_remote[0] = rank * numparticles; - mysize[0] = numparticles; - - // create a region - region_x = PDCregion_create(ndim, offset, mysize); - region_y = PDCregion_create(ndim, offset, mysize); - region_z = PDCregion_create(ndim, offset, mysize); - region_px = PDCregion_create(ndim, offset, mysize); - region_py = PDCregion_create(ndim, offset, mysize); - region_pz = PDCregion_create(ndim, offset, mysize); - region_id1 = PDCregion_create(ndim, offset, mysize); - region_id2 = PDCregion_create(ndim, offset, mysize); - - region_xx = PDCregion_create(ndim, offset_remote, mysize); - region_yy = PDCregion_create(ndim, offset_remote, mysize); - region_zz = PDCregion_create(ndim, offset_remote, mysize); - region_pxx = PDCregion_create(ndim, offset_remote, mysize); - region_pyy = PDCregion_create(ndim, offset_remote, mysize); - region_pzz = PDCregion_create(ndim, offset_remote, mysize); - region_id11 = PDCregion_create(ndim, offset_remote, mysize); - region_id22 = PDCregion_create(ndim, offset_remote, mysize); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCbuf_obj_map(&x[0], PDC_FLOAT, region_x, obj_xx, region_xx); - if (ret < 0) - LOG_ERROR("Array x PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &x[0], region_x, obj_xx, region_xx, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&y[0], PDC_FLOAT, region_y, obj_yy, region_yy); - if (ret < 0) - LOG_ERROR("Array y PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &y[0], region_y, obj_yy, region_yy, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&z[0], PDC_FLOAT, region_z, obj_zz, region_zz); - if (ret < 0) - LOG_ERROR("Array z PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &z[0], region_z, obj_zz, region_zz, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&px[0], PDC_FLOAT, region_px, obj_pxx, region_pxx); - if (ret < 0) - LOG_ERROR("Array px PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &px[0], region_px, obj_pxx, region_pxx, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&py[0], PDC_FLOAT, region_py, obj_pyy, region_pyy); - if (ret < 0) - LOG_ERROR("Array py PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &py[0], region_py, obj_pyy, region_pyy, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&pz[0], PDC_FLOAT, region_pz, obj_pzz, region_pzz); - if (ret < 0) - LOG_ERROR("Array pz PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &pz[0], region_pz, obj_pzz, region_pzz, 1, - DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&id1[0], PDC_INT, region_id1, obj_id11, region_id11); - if (ret < 0) - LOG_ERROR("Array id1 PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &id1[0], region_id1, obj_id11, - region_id11, 1, DECR_STATE, DATA_IN); - - ret = PDCbuf_obj_map(&id2[0], PDC_INT, region_id2, obj_id22, region_id22); - if (ret < 0) - LOG_ERROR("Array id2 PDCbuf_obj_map failed\n"); - ret = PDCbuf_map_transform_register("pdc_transform_decompress", &id2[0], region_id2, obj_id22, - region_id22, 1, DECR_STATE, DATA_IN); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to map with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCreg_obtain_lock(obj_xx, region_xx, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_xx\n"); - - ret = PDCreg_obtain_lock(obj_yy, region_yy, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_yy\n"); - - ret = PDCreg_obtain_lock(obj_zz, region_zz, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_zz\n"); - - ret = PDCreg_obtain_lock(obj_pxx, region_pxx, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pxx\n"); - - ret = PDCreg_obtain_lock(obj_pyy, region_pyy, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pyy\n"); - - ret = PDCreg_obtain_lock(obj_pzz, region_pzz, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pzz\n"); - - ret = PDCreg_obtain_lock(obj_id11, region_id11, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_id11\n"); - - ret = PDCreg_obtain_lock(obj_id22, region_id22, PDC_READ, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCreg_release_lock(obj_xx, region_xx, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_xx\n"); - - ret = PDCreg_release_lock(obj_yy, region_yy, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_yy\n"); - - ret = PDCreg_release_lock(obj_zz, region_zz, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_zz\n"); - - ret = PDCreg_release_lock(obj_pxx, region_pxx, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pxx\n"); - - ret = PDCreg_release_lock(obj_pyy, region_pyy, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pyy\n"); - - ret = PDCreg_release_lock(obj_pzz, region_pzz, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pzz\n"); - - ret = PDCreg_release_lock(obj_id11, region_id11, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_id11\n"); - - ret = PDCreg_release_lock(obj_id22, region_id22, PDC_READ); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to relese lock with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - ret = PDCbuf_obj_unmap(obj_xx, region_xx); - if (ret != SUCCEED) - LOG_ERROR("region xx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_yy, region_yy); - if (ret != SUCCEED) - LOG_ERROR("region yy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_zz, region_zz); - if (ret != SUCCEED) - LOG_ERROR("region zz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pxx, region_pxx); - if (ret != SUCCEED) - LOG_ERROR("region pxx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pyy, region_pyy); - if (ret != SUCCEED) - LOG_ERROR("region pyy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pzz, region_pzz); - if (ret != SUCCEED) - LOG_ERROR("region pzz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id11, region_id11); - if (ret != SUCCEED) - LOG_ERROR("region id11 unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id22, region_id22); - if (ret != SUCCEED) - LOG_ERROR("region id22 unmap failed\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to read data with %d ranks: %.5e\n", size, ht_total_sec); - - if (PDCobj_close(obj_xx) < 0) - LOG_ERROR("Failed to close obj_xx\n"); - - if (PDCobj_close(obj_yy) < 0) - LOG_ERROR("Failed to close object obj_yy\n"); - - if (PDCobj_close(obj_zz) < 0) - LOG_ERROR("Failed to close object obj_zz\n"); - - if (PDCobj_close(obj_pxx) < 0) - LOG_ERROR("Failed to close object obj_pxx\n"); - - if (PDCobj_close(obj_pyy) < 0) - LOG_ERROR("Failed to close object obj_pyy\n"); - - if (PDCobj_close(obj_pzz) < 0) - LOG_ERROR("Failed to close object obj_pzz\n"); - - if (PDCobj_close(obj_id11) < 0) - LOG_ERROR("Failed to close object obj_id11\n"); - - if (PDCobj_close(obj_id22) < 0) - LOG_ERROR("Failed to close object obj_id22\n"); - - if (PDCregion_close(region_x) < 0) - LOG_ERROR("Failed to close region region_x\n"); - - if (PDCregion_close(region_y) < 0) - LOG_ERROR("Failed to close region region_y\n"); - - if (PDCregion_close(region_z) < 0) - LOG_ERROR("Failed to close region region_z\n"); - - if (PDCregion_close(region_px) < 0) - LOG_ERROR("Failed to close region region_px\n"); - - if (PDCregion_close(region_py) < 0) - LOG_ERROR("Failed to close region region_py\n"); - - if (PDCobj_close(region_pz) < 0) - LOG_ERROR("Failed to close region region_pz\n"); - - if (PDCobj_close(region_id1) < 0) - LOG_ERROR("Failed to close region region_id1\n"); - - if (PDCobj_close(region_id2) < 0) - LOG_ERROR("Failed to close region region_id2\n"); - - if (PDCregion_close(region_xx) < 0) - LOG_ERROR("Failed to close region region_xx\n"); - - if (PDCregion_close(region_yy) < 0) - LOG_ERROR("Failed to close region region_yy\n"); - - if (PDCregion_close(region_zz) < 0) - LOG_ERROR("Failed to close region region_zz\n"); - - if (PDCregion_close(region_pxx) < 0) - LOG_ERROR("Failed to close region region_pxx\n"); - - if (PDCregion_close(region_pyy) < 0) - LOG_ERROR("Failed to close region region_pyy\n"); - - if (PDCregion_close(region_pzz) < 0) - LOG_ERROR("Failed to close region region_pzz\n"); - - if (PDCobj_close(region_id11) < 0) - LOG_ERROR("Failed to close region region_id11\n"); - - if (PDCobj_close(region_id22) < 0) - LOG_ERROR("Failed to close region region_id22\n"); - - // close a container - if (PDCcont_close(cont_id) < 0) - LOG_ERROR("Failed to close container c1\n"); - - if (PDCclose(pdc_id) < 0) - LOG_ERROR("Failed to close PDC\n"); - - free(x); - free(y); - free(z); - free(px); - free(py); - free(pz); - free(id1); - free(id2); - free(offset); - free(offset_remote); - free(mysize); - -#ifdef ENABLE_MPI - MPI_Finalize(); -#endif - - return 0; -} diff --git a/src/tests/deprecated/obj_transformation.c b/src/tests/deprecated/obj_transformation.c deleted file mode 100644 index ebba0f16d..000000000 --- a/src/tests/deprecated/obj_transformation.c +++ /dev/null @@ -1,545 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "pdc.h" - -#define NPARTICLES 8388608 - -double -uniform_random_number() -{ - return (((double)rand()) / ((double)(RAND_MAX))); -} - -void -print_usage() -{ - LOG_JUST_PRINT("Usage: srun -n ./vpicio #particles\n"); -} - -int -main(int argc, char **argv) -{ - int rank = 0, size = 1; - pdcid_t pdc_id, cont_prop, cont_id; - pdcid_t obj_prop_xx, obj_prop_yy, obj_prop_zz, obj_prop_pxx, obj_prop_pyy, obj_prop_pzz, obj_prop_id11, - obj_prop_id22; - pdcid_t obj_xx, obj_yy, obj_zz, obj_pxx, obj_pyy, obj_pzz, obj_id11, obj_id22; - pdcid_t region_x, region_y, region_z, region_px, region_py, region_pz, region_id1, region_id2; - pdcid_t region_xx, region_yy, region_zz, region_pxx, region_pyy, region_pzz, region_id11, region_id22; - perr_t ret; - struct timeval ht_total_start; - struct timeval ht_total_end; - long long ht_total_elapsed; - double ht_total_sec; - float * x, *y, *z; - float * px, *py, *pz; - int * id1, *id2; - int x_dim = 64; - int y_dim = 64; - int z_dim = 64; - uint64_t numparticles, i; - uint64_t dims[1]; - int ndim = 1; - uint64_t * offset; - uint64_t * offset_remote; - uint64_t * mysize; - -#ifdef ENABLE_MPI - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); -#endif - - numparticles = NPARTICLES; - dims[0] = numparticles; - - x = (float *)malloc(numparticles * sizeof(float)); - y = (float *)malloc(numparticles * sizeof(float)); - z = (float *)malloc(numparticles * sizeof(float)); - - px = (float *)malloc(numparticles * sizeof(float)); - py = (float *)malloc(numparticles * sizeof(float)); - pz = (float *)malloc(numparticles * sizeof(float)); - - id1 = (int *)malloc(numparticles * sizeof(int)); - id2 = (int *)malloc(numparticles * sizeof(int)); - - // create a pdc - pdc_id = PDCinit("pdc"); - - // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc_id); - if (cont_prop <= 0) - LOG_ERROR("Failed to create container property"); - - // create a container - cont_id = PDCcont_create("c1", cont_prop); - if (cont_id <= 0) - LOG_ERROR("Failed to create container"); - - // create an object property - obj_prop_xx = PDCprop_create(PDC_OBJ_CREATE, pdc_id); - - PDCprop_set_obj_dims(obj_prop_xx, 1, dims); - PDCprop_set_obj_type(obj_prop_xx, PDC_FLOAT); - PDCprop_set_obj_time_step(obj_prop_xx, 0); - PDCprop_set_obj_user_id(obj_prop_xx, getuid()); - PDCprop_set_obj_app_name(obj_prop_xx, "VPICIO"); - PDCprop_set_obj_tags(obj_prop_xx, "tag0=1"); - - obj_prop_yy = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_yy, PDC_FLOAT); - - obj_prop_zz = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_zz, PDC_FLOAT); - - obj_prop_pxx = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pxx, PDC_FLOAT); - - obj_prop_pyy = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pyy, PDC_FLOAT); - - obj_prop_pzz = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pzz, PDC_FLOAT); - - obj_prop_id11 = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_id11, PDC_INT); - - obj_prop_id22 = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_id22, PDC_INT); - -#ifdef ENABLE_MPI - obj_xx = PDCobj_create_mpi(cont_id, "obj-var-xx", obj_prop_xx, 0, MPI_COMM_WORLD); - if (obj_xx == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-xx"); - exit(-1); - } - - obj_yy = PDCobj_create_mpi(cont_id, "obj-var-yy", obj_prop_yy, 0, MPI_COMM_WORLD); - if (obj_yy == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-yy"); - exit(-1); - } - obj_zz = PDCobj_create_mpi(cont_id, "obj-var-zz", obj_prop_zz, 0, MPI_COMM_WORLD); - if (obj_zz == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-zz"); - exit(-1); - } - obj_pxx = PDCobj_create_mpi(cont_id, "obj-var-pxx", obj_prop_pxx, 0, MPI_COMM_WORLD); - if (obj_pxx == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pxx"); - exit(-1); - } - obj_pyy = PDCobj_create_mpi(cont_id, "obj-var-pyy", obj_prop_pyy, 0, MPI_COMM_WORLD); - if (obj_pyy == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pyy"); - exit(-1); - } - obj_pzz = PDCobj_create_mpi(cont_id, "obj-var-pzz", obj_prop_pzz, 0, MPI_COMM_WORLD); - if (obj_pzz == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pzz"); - exit(-1); - } - obj_id11 = PDCobj_create_mpi(cont_id, "id11", obj_prop_id11, 0, MPI_COMM_WORLD); - if (obj_id11 == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj_id11"); - exit(-1); - } - obj_id22 = PDCobj_create_mpi(cont_id, "id22", obj_prop_id22, 0, MPI_COMM_WORLD); - if (obj_id22 == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj_id22"); - exit(-1); - } -#endif - - offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset_remote = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - mysize = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset[0] = 0; - offset_remote[0] = rank * numparticles; - mysize[0] = numparticles; - - // create a region - region_x = PDCregion_create(ndim, offset, mysize); - region_y = PDCregion_create(ndim, offset, mysize); - region_z = PDCregion_create(ndim, offset, mysize); - region_px = PDCregion_create(ndim, offset, mysize); - region_py = PDCregion_create(ndim, offset, mysize); - region_pz = PDCregion_create(ndim, offset, mysize); - region_id1 = PDCregion_create(ndim, offset, mysize); - region_id2 = PDCregion_create(ndim, offset, mysize); - - region_xx = PDCregion_create(ndim, offset_remote, mysize); - region_yy = PDCregion_create(ndim, offset_remote, mysize); - region_zz = PDCregion_create(ndim, offset_remote, mysize); - region_pxx = PDCregion_create(ndim, offset_remote, mysize); - region_pyy = PDCregion_create(ndim, offset_remote, mysize); - region_pzz = PDCregion_create(ndim, offset_remote, mysize); - region_id11 = PDCregion_create(ndim, offset_remote, mysize); - region_id22 = PDCregion_create(ndim, offset_remote, mysize); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - -#if 0 - ret = PDCreg_transform_register("pdc_transform_compress", &x[0], region_x, obj_xx, region_xx, 0, INCR_STATE, DATA_OUT); - ret = PDCbuf_transform_register("pdc_transform_compress", &x[0], region_x, obj_xx, region_xx, 0, INCR_STATE, DATA_OUT); - ret = PDCobj_transform_register("pdc_transform_compress", &x[0], region_x, obj_xx, region_xx, 0, INCR_STATE, DATA_OUT); -#else - ret = PDCbuf_map_transform_register("pdc_transform_increment", &x[0], region_x, obj_xx, region_xx, 0, - INCR_STATE, DATA_OUT); - ret = PDCbuf_map_transform_register("pdc_transform_compress", &x[0], region_x, obj_xx, region_xx, 0, - INCR_STATE, DATA_OUT); -#endif - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(1) failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_decompress", NULL, region_x, obj_xx, region_xx, 1, - DECR_STATE, DATA_IN); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(2) failed\n"); - - ret = PDCbuf_obj_map(&x[0], PDC_FLOAT, region_x, obj_xx, region_xx); - if (ret < 0) - LOG_ERROR("Array x PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&y[0], PDC_FLOAT, region_y, obj_yy, region_yy); - if (ret < 0) - LOG_ERROR("Array y PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&z[0], PDC_FLOAT, region_z, obj_zz, region_zz); - if (ret < 0) - LOG_ERROR("Array z PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&px[0], PDC_FLOAT, region_px, obj_pxx, region_pxx); - if (ret < 0) - LOG_ERROR("Array px PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&py[0], PDC_FLOAT, region_py, obj_pyy, region_pyy); - if (ret < 0) - LOG_ERROR("Array py PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&pz[0], PDC_FLOAT, region_pz, obj_pzz, region_pzz); - if (ret < 0) - LOG_ERROR("Array pz PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&id1[0], PDC_INT, region_id1, obj_id11, region_id11); - if (ret < 0) - LOG_ERROR("Array id1 PDCbuf_obj_map failed\n"); - - ret = PDCbuf_obj_map(&id2[0], PDC_INT, region_id2, obj_id22, region_id22); - if (ret < 0) - LOG_ERROR("Array id2 PDCbuf_obj_map failed\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to map with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCreg_obtain_lock(obj_xx, region_xx, PDC_WRITE, PDC_NOBLOCK); - if (ret < 0) - LOG_ERROR("Failed to obtain lock for region_xx\n"); - - ret = PDCreg_obtain_lock(obj_yy, region_yy, PDC_WRITE, PDC_NOBLOCK); - if (ret < 0) - LOG_ERROR("Failed to obtain lock for region_yy\n"); - - ret = PDCreg_obtain_lock(obj_zz, region_zz, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_zz\n"); - - ret = PDCreg_obtain_lock(obj_pxx, region_pxx, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pxx\n"); - - ret = PDCreg_obtain_lock(obj_pyy, region_pyy, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pyy\n"); - - ret = PDCreg_obtain_lock(obj_pzz, region_pzz, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_pzz\n"); - - ret = PDCreg_obtain_lock(obj_id11, region_id11, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_id11\n"); - - ret = PDCreg_obtain_lock(obj_id22, region_id22, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("Failed to obtain lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); - - for (i = 0; i < numparticles; i++) { - id1[i] = i; - id2[i] = i * 2; - x[i] = uniform_random_number() * x_dim; - y[i] = uniform_random_number() * y_dim; - z[i] = ((float)id1[i] / numparticles) * z_dim; - px[i] = uniform_random_number() * x_dim; - py[i] = uniform_random_number() * y_dim; - pz[i] = ((float)id2[i] / numparticles) * z_dim; - } - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - x[0] = 1.0; - x[1] = 2.0; - - ret = PDCreg_release_lock(obj_xx, region_xx, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_xx\n"); - - ret = PDCreg_release_lock(obj_yy, region_yy, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_yy\n"); - - ret = PDCreg_release_lock(obj_zz, region_zz, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_zz\n"); - - ret = PDCreg_release_lock(obj_pxx, region_pxx, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pxx\n"); - - ret = PDCreg_release_lock(obj_pyy, region_pyy, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pyy\n"); - - ret = PDCreg_release_lock(obj_pzz, region_pzz, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_pzz\n"); - - ret = PDCreg_release_lock(obj_id11, region_id11, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_id11\n"); - - ret = PDCreg_release_lock(obj_id22, region_id22, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("Failed to release lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - ret = PDCbuf_obj_unmap(obj_xx, region_xx); - if (ret != SUCCEED) - LOG_ERROR("region xx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_yy, region_yy); - if (ret != SUCCEED) - LOG_ERROR("region yy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_zz, region_zz); - if (ret != SUCCEED) - LOG_ERROR("region zz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pxx, region_pxx); - if (ret != SUCCEED) - LOG_ERROR("region pxx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pyy, region_pyy); - if (ret != SUCCEED) - LOG_ERROR("region pyy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pzz, region_pzz); - if (ret != SUCCEED) - LOG_ERROR("region pzz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id11, region_id11); - if (ret != SUCCEED) - LOG_ERROR("region id11 unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id22, region_id22); - if (ret != SUCCEED) - LOG_ERROR("region id22 unmap failed\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - if (PDCobj_close(obj_xx) < 0) - LOG_ERROR("Failed to close obj_xx\n"); - - if (PDCobj_close(obj_yy) < 0) - LOG_ERROR("Failed to close object obj_yy\n"); - - if (PDCobj_close(obj_zz) < 0) - LOG_ERROR("Failed to close object obj_zz\n"); - - if (PDCobj_close(obj_pxx) < 0) - LOG_ERROR("Failed to close object obj_pxx\n"); - - if (PDCobj_close(obj_pyy) < 0) - LOG_ERROR("Failed to close object obj_pyy\n"); - - if (PDCobj_close(obj_pzz) < 0) - LOG_ERROR("Failed to close object obj_pzz\n"); - - if (PDCobj_close(obj_id11) < 0) - LOG_ERROR("Failed to close object obj_id11\n"); - - if (PDCobj_close(obj_id22) < 0) - LOG_ERROR("Failed to close object obj_id22\n"); - - if (PDCprop_close(obj_prop_xx) < 0) - LOG_ERROR("Failed to close obj property obj_prop_xx\n"); - - if (PDCprop_close(obj_prop_yy) < 0) - LOG_ERROR("Failed to close obj property obj_prop_yy\n"); - - if (PDCprop_close(obj_prop_zz) < 0) - LOG_ERROR("Failed to close obj property obj_prop_zz\n"); - - if (PDCprop_close(obj_prop_pxx) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pxx\n"); - - if (PDCprop_close(obj_prop_pyy) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pyy\n"); - - if (PDCprop_close(obj_prop_pzz) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pzz\n"); - - if (PDCprop_close(obj_prop_id11) < 0) - LOG_ERROR("Failed to close obj property obj_prop_id11\n"); - - if (PDCprop_close(obj_prop_id22) < 0) - LOG_ERROR("Failed to close obj property obj_prop_id22\n"); - - if (PDCregion_close(region_x) < 0) - LOG_ERROR("Failed to close region region_x\n"); - - if (PDCregion_close(region_y) < 0) - LOG_ERROR("Failed to close region region_y\n"); - - if (PDCregion_close(region_z) < 0) - LOG_ERROR("Failed to close region region_z\n"); - - if (PDCregion_close(region_px) < 0) - LOG_ERROR("Failed to close region region_px\n"); - - if (PDCregion_close(region_py) < 0) - LOG_ERROR("Failed to close region region_py\n"); - - if (PDCobj_close(region_pz) < 0) - LOG_ERROR("Failed to close region region_pz\n"); - - if (PDCobj_close(region_id1) < 0) - LOG_ERROR("Failed to close region region_id1\n"); - - if (PDCobj_close(region_id2) < 0) - LOG_ERROR("Failed to close region region_id2\n"); - - if (PDCregion_close(region_xx) < 0) - LOG_ERROR("Failed to close region region_xx\n"); - - if (PDCregion_close(region_yy) < 0) - LOG_ERROR("Failed to close region region_yy\n"); - - if (PDCregion_close(region_zz) < 0) - LOG_ERROR("Failed to close region region_zz\n"); - - if (PDCregion_close(region_pxx) < 0) - LOG_ERROR("Failed to close region region_pxx\n"); - - if (PDCregion_close(region_pyy) < 0) - LOG_ERROR("Failed to close region region_pyy\n"); - - if (PDCregion_close(region_pzz) < 0) - LOG_ERROR("Failed to close region region_pzz\n"); - - if (PDCobj_close(region_id11) < 0) - LOG_ERROR("Failed to close region region_id11\n"); - - if (PDCobj_close(region_id22) < 0) - LOG_ERROR("Failed to close region region_id22\n"); - - // close a container - if (PDCcont_close(cont_id) < 0) - LOG_ERROR("Failed to close container c1\n"); - - // close a container property - if (PDCprop_close(cont_prop) < 0) - LOG_ERROR("Failed to close property"); - - if (PDCclose(pdc_id) < 0) - LOG_ERROR("Failed to close PDC\n"); - - free(x); - free(y); - free(z); - free(px); - free(py); - free(pz); - free(id1); - free(id2); - -#ifdef ENABLE_MPI - MPI_Finalize(); -#endif - - return 0; -} diff --git a/src/tests/deprecated/vpicio_v2.c b/src/tests/deprecated/vpicio_v2.c deleted file mode 100644 index d540a881f..000000000 --- a/src/tests/deprecated/vpicio_v2.c +++ /dev/null @@ -1,573 +0,0 @@ -/* - * Copyright Notice for - * Proactive Data Containers (PDC) Software Library and Utilities - * ----------------------------------------------------------------------------- - - *** Copyright Notice *** - - * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the - * University of California, through Lawrence Berkeley National Laboratory, - * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF - * Group (subject to receipt of any required approvals from the U.S. Dept. of - * Energy). All rights reserved. - - * If you have questions about your rights to use or distribute this software, - * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. - - * NOTICE. This Software was developed under funding from the U.S. Department of - * Energy and the U.S. Government consequently retains certain rights. As such, the - * U.S. Government has been granted for itself and others acting on its behalf a - * paid-up, nonexclusive, irrevocable, worldwide license in the Software to - * reproduce, distribute copies to the public, prepare derivative works, and - * perform publicly and display publicly, and to permit other to do so. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "pdc.h" - -#define NPARTICLES 8388608 - -double -uniform_random_number() -{ - return (((double)rand()) / ((double)(RAND_MAX))); -} - -void -print_usage() -{ - LOG_JUST_PRINT("Usage: srun -n ./vpicio #particles\n"); -} - -int -main(int argc, char **argv) -{ - int rank = 0, size = 1; - pdcid_t pdc_id, cont_prop, cont_id; - pdcid_t obj_prop_xx, obj_prop_yy, obj_prop_zz, obj_prop_pxx, obj_prop_pyy, obj_prop_pzz, obj_prop_id11, - obj_prop_id22; - pdcid_t obj_xx, obj_yy, obj_zz, obj_pxx, obj_pyy, obj_pzz, obj_id11, obj_id22; - pdcid_t region_x, region_y, region_z, region_px, region_py, region_pz, region_id1, region_id2; - pdcid_t region_xx, region_yy, region_zz, region_pxx, region_pyy, region_pzz, region_id11, region_id22; - perr_t ret; -#ifdef ENABLE_MPI - MPI_Comm comm; -#endif - struct timeval ht_total_start; - struct timeval ht_total_end; - long long ht_total_elapsed; - double ht_total_sec; - float * x, *y, *z; - float * px, *py, *pz; - int * id1, *id2; - int x_dim = 64; - int y_dim = 64; - int z_dim = 64; - uint64_t numparticles, i; - uint64_t dims[1]; - int ndim = 1; - uint64_t * offset; - uint64_t * offset_remote; - uint64_t * mysize; - -#ifdef ENABLE_MPI - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_dup(MPI_COMM_WORLD, &comm); -#endif - - numparticles = NPARTICLES; - dims[0] = numparticles; - - x = (float *)malloc(numparticles * sizeof(float)); - y = (float *)malloc(numparticles * sizeof(float)); - z = (float *)malloc(numparticles * sizeof(float)); - - px = (float *)malloc(numparticles * sizeof(float)); - py = (float *)malloc(numparticles * sizeof(float)); - pz = (float *)malloc(numparticles * sizeof(float)); - - id1 = (int *)malloc(numparticles * sizeof(int)); - id2 = (int *)malloc(numparticles * sizeof(int)); - - // create a pdc - pdc_id = PDCinit("pdc"); - - // create a container property - cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc_id); - if (cont_prop <= 0) - LOG_ERROR("Failed to create container property"); - - // create a container - cont_id = PDCcont_create_col("c1", cont_prop); - if (cont_id <= 0) - LOG_ERROR("Failed to create container"); - - // create an object property - obj_prop_xx = PDCprop_create(PDC_OBJ_CREATE, pdc_id); - - PDCprop_set_obj_dims(obj_prop_xx, 1, dims); - PDCprop_set_obj_type(obj_prop_xx, PDC_FLOAT); - PDCprop_set_obj_time_step(obj_prop_xx, 0); - PDCprop_set_obj_user_id(obj_prop_xx, getuid()); - PDCprop_set_obj_app_name(obj_prop_xx, "VPICIO"); - PDCprop_set_obj_tags(obj_prop_xx, "tag0=1"); - - obj_prop_yy = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_yy, PDC_FLOAT); - - obj_prop_zz = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_zz, PDC_FLOAT); - - obj_prop_pxx = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pxx, PDC_FLOAT); - - obj_prop_pyy = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pyy, PDC_FLOAT); - - obj_prop_pzz = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_pzz, PDC_FLOAT); - - obj_prop_id11 = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_id11, PDC_INT); - - obj_prop_id22 = PDCprop_obj_dup(obj_prop_xx); - PDCprop_set_obj_type(obj_prop_id22, PDC_INT); - -#ifdef ENABLE_MPI - obj_xx = PDCobj_create_mpi(cont_id, "obj-var-xx", obj_prop_xx, 0, comm); - if (obj_xx == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-xx"); - exit(-1); - } - - obj_yy = PDCobj_create_mpi(cont_id, "obj-var-yy", obj_prop_yy, 0, comm); - if (obj_yy == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-yy"); - exit(-1); - } - obj_zz = PDCobj_create_mpi(cont_id, "obj-var-zz", obj_prop_zz, 0, comm); - if (obj_zz == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-zz"); - exit(-1); - } - obj_pxx = PDCobj_create_mpi(cont_id, "obj-var-pxx", obj_prop_pxx, 0, comm); - if (obj_pxx == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pxx"); - exit(-1); - } - obj_pyy = PDCobj_create_mpi(cont_id, "obj-var-pyy", obj_prop_pyy, 0, comm); - if (obj_pyy == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pyy"); - exit(-1); - } - obj_pzz = PDCobj_create_mpi(cont_id, "obj-var-pzz", obj_prop_pzz, 0, comm); - if (obj_pzz == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj-var-pzz"); - exit(-1); - } - - obj_id11 = PDCobj_create_mpi(cont_id, "id11", obj_prop_id11, 0, comm); - if (obj_id11 == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj_id11"); - exit(-1); - } - obj_id22 = PDCobj_create_mpi(cont_id, "id22", obj_prop_id22, 0, comm); - if (obj_id22 == 0) { - LOG_ERROR("Error getting an object id of %s from server, exit...\n", "obj_id22"); - exit(-1); - } -#endif - - offset = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset_remote = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - mysize = (uint64_t *)malloc(sizeof(uint64_t) * ndim); - offset[0] = 0; - offset_remote[0] = rank * numparticles; - mysize[0] = numparticles; - - // create a region - region_x = PDCregion_create(ndim, offset, mysize); - region_y = PDCregion_create(ndim, offset, mysize); - region_z = PDCregion_create(ndim, offset, mysize); - region_px = PDCregion_create(ndim, offset, mysize); - region_py = PDCregion_create(ndim, offset, mysize); - region_pz = PDCregion_create(ndim, offset, mysize); - region_id1 = PDCregion_create(ndim, offset, mysize); - region_id2 = PDCregion_create(ndim, offset, mysize); - - region_xx = PDCregion_create(ndim, offset_remote, mysize); - region_yy = PDCregion_create(ndim, offset_remote, mysize); - region_zz = PDCregion_create(ndim, offset_remote, mysize); - region_pxx = PDCregion_create(ndim, offset_remote, mysize); - region_pyy = PDCregion_create(ndim, offset_remote, mysize); - region_pzz = PDCregion_create(ndim, offset_remote, mysize); - region_id11 = PDCregion_create(ndim, offset_remote, mysize); - region_id22 = PDCregion_create(ndim, offset_remote, mysize); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCbuf_obj_map(&x[0], PDC_FLOAT, region_x, obj_xx, region_xx); - if (ret < 0) - LOG_ERROR("Array x PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &x[0], region_x, obj_xx, region_xx, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(1) failed\n"); - - ret = PDCbuf_obj_map(&y[0], PDC_FLOAT, region_y, obj_yy, region_yy); - if (ret < 0) - LOG_ERROR("Array y PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &y[0], region_y, obj_yy, region_yy, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(2) failed\n"); - - ret = PDCbuf_obj_map(&z[0], PDC_FLOAT, region_z, obj_zz, region_zz); - if (ret < 0) - LOG_ERROR("Array z PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &z[0], region_z, obj_zz, region_zz, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(3) failed\n"); - - ret = PDCbuf_obj_map(&px[0], PDC_FLOAT, region_px, obj_pxx, region_pxx); - if (ret < 0) - LOG_ERROR("Array px PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &px[0], region_px, obj_pxx, region_pxx, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(4) failed\n"); - - ret = PDCbuf_obj_map(&py[0], PDC_FLOAT, region_py, obj_pyy, region_pyy); - if (ret < 0) - LOG_ERROR("Array py PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &py[0], region_py, obj_pyy, region_pyy, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(5) failed\n"); - - ret = PDCbuf_obj_map(&pz[0], PDC_FLOAT, region_pz, obj_pzz, region_pzz); - if (ret < 0) - LOG_ERROR("Array pz PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &pz[0], region_pz, obj_pzz, region_pzz, 0, - INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(6) failed\n"); - - ret = PDCbuf_obj_map(&id1[0], PDC_INT, region_id1, obj_id11, region_id11); - if (ret < 0) - LOG_ERROR("Array id1 PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &id1[0], region_id1, obj_id11, region_id11, - 0, INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(7) failed\n"); - - ret = PDCbuf_obj_map(&id2[0], PDC_INT, region_id2, obj_id22, region_id22); - if (ret < 0) - LOG_ERROR("Array id2 PDCbuf_obj_map failed\n"); - - ret = PDCbuf_map_transform_register("pdc_transform_compress", &id2[0], region_id2, obj_id22, region_id22, - 0, INCR_STATE, DATA_OUT); - if (ret < 0) - LOG_ERROR("PDCobj_transform_register(8) failed\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to map with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCreg_obtain_lock(obj_xx, region_xx, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_xx\n"); - - ret = PDCreg_obtain_lock(obj_yy, region_yy, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_yy\n"); - - ret = PDCreg_obtain_lock(obj_zz, region_zz, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_zz\n"); - - ret = PDCreg_obtain_lock(obj_pxx, region_pxx, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_pxx\n"); - - ret = PDCreg_obtain_lock(obj_pyy, region_pyy, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_pyy\n"); - - ret = PDCreg_obtain_lock(obj_pzz, region_pzz, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_pzz\n"); - - ret = PDCreg_obtain_lock(obj_id11, region_id11, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_id11\n"); - - ret = PDCreg_obtain_lock(obj_id22, region_id22, PDC_WRITE, PDC_NOBLOCK); - if (ret != SUCCEED) - LOG_ERROR("failed to obtain lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to lock with %d ranks: %.5e\n", size, ht_total_sec); - - for (i = 0; i < numparticles; i++) { - id1[i] = i; - id2[i] = i * 2; - x[i] = uniform_random_number() * x_dim; - y[i] = uniform_random_number() * y_dim; - z[i] = ((float)id1[i] / numparticles) * z_dim; - px[i] = uniform_random_number() * x_dim; - py[i] = uniform_random_number() * y_dim; - pz[i] = ((float)id2[i] / numparticles) * z_dim; - } - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_start, 0); - - ret = PDCreg_release_lock(obj_xx, region_xx, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_xx\n"); - - ret = PDCreg_release_lock(obj_yy, region_yy, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_yy\n"); - - ret = PDCreg_release_lock(obj_zz, region_zz, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_zz\n"); - - ret = PDCreg_release_lock(obj_pxx, region_pxx, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_pxx\n"); - - ret = PDCreg_release_lock(obj_pyy, region_pyy, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_pyy\n"); - - ret = PDCreg_release_lock(obj_pzz, region_pzz, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_pzz\n"); - - ret = PDCreg_release_lock(obj_id11, region_id11, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_id11\n"); - - ret = PDCreg_release_lock(obj_id22, region_id22, PDC_WRITE); - if (ret != SUCCEED) - LOG_ERROR("failed to release lock for region_id22\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - gettimeofday(&ht_total_end, 0); - ht_total_elapsed = (ht_total_end.tv_sec - ht_total_start.tv_sec) * 1000000LL + ht_total_end.tv_usec - - ht_total_start.tv_usec; - ht_total_sec = ht_total_elapsed / 1000000.0; - if (rank == 0) - LOG_INFO("Time to update data with %d ranks: %.5e\n", size, ht_total_sec); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - ret = PDCbuf_obj_unmap(obj_xx, region_xx); - if (ret != SUCCEED) - LOG_ERROR("region xx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_yy, region_yy); - if (ret != SUCCEED) - LOG_ERROR("region yy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_zz, region_zz); - if (ret != SUCCEED) - LOG_ERROR("region zz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pxx, region_pxx); - if (ret != SUCCEED) - LOG_ERROR("region pxx unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pyy, region_pyy); - if (ret != SUCCEED) - LOG_ERROR("region pyy unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_pzz, region_pzz); - if (ret != SUCCEED) - LOG_ERROR("region pzz unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id11, region_id11); - if (ret != SUCCEED) - LOG_ERROR("region id11 unmap failed\n"); - - ret = PDCbuf_obj_unmap(obj_id22, region_id22); - if (ret != SUCCEED) - LOG_ERROR("region id22 unmap failed\n"); - -#ifdef ENABLE_MPI - MPI_Barrier(MPI_COMM_WORLD); -#endif - - if (PDCobj_close(obj_xx) < 0) - LOG_ERROR("Failed to close obj_xx\n"); - - if (PDCobj_close(obj_yy) < 0) - LOG_ERROR("Failed to close object obj_yy\n"); - - if (PDCobj_close(obj_zz) < 0) - LOG_ERROR("Failed to close object obj_zz\n"); - - if (PDCobj_close(obj_pxx) < 0) - LOG_ERROR("Failed to close object obj_pxx\n"); - - if (PDCobj_close(obj_pyy) < 0) - LOG_ERROR("Failed to close object obj_pyy\n"); - - if (PDCobj_close(obj_pzz) < 0) - LOG_ERROR("Failed to close object obj_pzz\n"); - - if (PDCobj_close(obj_id11) < 0) - LOG_ERROR("Failed to close object obj_id11\n"); - - if (PDCobj_close(obj_id22) < 0) - LOG_ERROR("Failed to close object obj_id22\n"); - - if (PDCprop_close(obj_prop_xx) < 0) - LOG_ERROR("Failed to close obj property obj_prop_xx\n"); - - if (PDCprop_close(obj_prop_yy) < 0) - LOG_ERROR("Failed to close obj property obj_prop_yy\n"); - - if (PDCprop_close(obj_prop_zz) < 0) - LOG_ERROR("Failed to close obj property obj_prop_zz\n"); - - if (PDCprop_close(obj_prop_pxx) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pxx\n"); - - if (PDCprop_close(obj_prop_pyy) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pyy\n"); - - if (PDCprop_close(obj_prop_pzz) < 0) - LOG_ERROR("Failed to close obj property obj_prop_pzz\n"); - - if (PDCprop_close(obj_prop_id11) < 0) - LOG_ERROR("Failed to close obj property obj_prop_id11\n"); - - if (PDCprop_close(obj_prop_id22) < 0) - LOG_ERROR("Failed to close obj property obj_prop_id22\n"); - - if (PDCregion_close(region_x) < 0) - LOG_ERROR("Failed to close region region_x\n"); - - if (PDCregion_close(region_y) < 0) - LOG_ERROR("Failed to close region region_y\n"); - - if (PDCregion_close(region_z) < 0) - LOG_ERROR("Failed to close region region_z\n"); - - if (PDCregion_close(region_px) < 0) - LOG_ERROR("Failed to close region region_px\n"); - - if (PDCregion_close(region_py) < 0) - LOG_ERROR("Failed to close region region_py\n"); - - if (PDCobj_close(region_pz) < 0) - LOG_ERROR("Failed to close region region_pz\n"); - - if (PDCobj_close(region_id1) < 0) - LOG_ERROR("Failed to close region region_id1\n"); - - if (PDCobj_close(region_id2) < 0) - LOG_ERROR("Failed to close region region_id2\n"); - - if (PDCregion_close(region_xx) < 0) - LOG_ERROR("Failed to close region region_xx\n"); - - if (PDCregion_close(region_yy) < 0) - LOG_ERROR("Failed to close region region_yy\n"); - - if (PDCregion_close(region_zz) < 0) - LOG_ERROR("Failed to close region region_zz\n"); - - if (PDCregion_close(region_pxx) < 0) - LOG_ERROR("Failed to close region region_pxx\n"); - - if (PDCregion_close(region_pyy) < 0) - LOG_ERROR("Failed to close region region_pyy\n"); - - if (PDCregion_close(region_pzz) < 0) - LOG_ERROR("Failed to close region region_pzz\n"); - - if (PDCobj_close(region_id11) < 0) - LOG_ERROR("Failed to close region region_id11\n"); - - if (PDCobj_close(region_id22) < 0) - LOG_ERROR("Failed to close region region_id22\n"); - - // close a container - if (PDCcont_close(cont_id) < 0) - LOG_ERROR("Failed to close container c1\n"); - - // close a container property - if (PDCprop_close(cont_prop) < 0) - LOG_ERROR("Failed to close property"); - - if (PDCclose(pdc_id) < 0) - LOG_ERROR("Failed to close PDC\n"); - - free(offset); - free(offset_remote); - free(mysize); - free(x); - free(y); - free(z); - free(px); - free(py); - free(pz); - free(id1); - free(id2); - -#ifdef ENABLE_MPI - MPI_Finalize(); -#endif - - return 0; -} diff --git a/src/tests/misc/bdcats.c b/src/tests/misc/bdcats.c index 9b3b593c2..bd46417d1 100644 --- a/src/tests/misc/bdcats.c +++ b/src/tests/misc/bdcats.c @@ -32,8 +32,21 @@ #include #include #include "pdc.h" +#include +#include +#include "/pscratch/sd/n/nlewi26/src/work_space/source/pdc/pi.h" -#define NPARTICLES 8388608 +#define dLEAP 2 +#define PRECISION_INIT 2 +#define PRECISION_HEX 20000000 + +#define WARP_SIZE 32 +#define WARPS_PER_BLOCK 2 +#define THREADS_PER_BLOCK (WARPS_PER_BLOCK * WARP_SIZE) +#define BLOCKS_NUM ((PRECISION_HEX / dLEAP / THREADS_PER_BLOCK) + 1) + +#define NUM_ITERATIONS 1 +#define NPARTICLES 8388608 double uniform_random_number() @@ -78,8 +91,8 @@ main(int argc, char **argv) sleeptime = atoi(argv[3]); if (rank == 0) - LOG_INFO("Reading %" PRIu64 " particles per rank for %d steps with %d sec sleep time.\n", - numparticles, steps, sleeptime); + LOG_WARNING("Reading %" PRIu64 " particles per rank for %d steps with %d sec sleep time.\n", + numparticles, steps, sleeptime); dims[0] = numparticles * size; @@ -99,9 +112,13 @@ main(int argc, char **argv) pdc_id = PDCinit("pdc"); // create a container +#ifdef ENABLE_MPI + cont_id = PDCcont_open_col("c1", pdc_id); +#else cont_id = PDCcont_open("c1", pdc_id); +#endif if (cont_id <= 0) { - LOG_ERROR("Failed to create container"); + LOG_ERROR("Failed to open container"); return FAIL; } @@ -118,7 +135,7 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); if (rank == 0) - LOG_INFO("\n#Step %d\n", iter); + LOG_WARNING("\n#Step %d\n", iter); t0 = MPI_Wtime(); #endif for (int i = 0; i < 8; i++) { @@ -138,7 +155,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Obj open time: %.5e\n", t1 - t0); + LOG_WARNING("Obj open time: %.5e\n", t1 - t0); #endif for (int i = 0; i < 8; i++) { @@ -154,7 +171,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer create time: %.5e\n", t0 - t1); + LOG_WARNING("Transfer create time: %.5e\n", t0 - t1); #endif #ifdef ENABLE_MPI @@ -170,15 +187,20 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer start time: %.5e\n", t1 - t0); + LOG_WARNING("Transfer start time: %.5e\n", t1 - t0); #endif // Emulate compute with sleep if (iter != steps - 1) { if (rank == 0) - LOG_INFO("Sleep start: %llu.00\n", sleeptime); - sleep(sleeptime); + LOG_WARNING("Sleep start: %llu.00\n", sleeptime); + double loop_start = MPI_Wtime(); + sleep(40); + double loop_end = MPI_Wtime(); + if (rank == 0) { + LOG_WARNING("\nTotal time for %d iterations: %f s\n", NUM_ITERATIONS, loop_end - loop_start); + } if (rank == 0) - LOG_INFO("Sleep end: %llu.00\n", sleeptime); + LOG_WARNING("Sleep end: %llu.00\n", sleeptime); } #ifdef ENABLE_MPI @@ -190,22 +212,11 @@ main(int argc, char **argv) LOG_ERROR("Failed to transfer wait all\n"); return FAIL; } - - // Verify data of id and q - if (id[0] != rank + iter || q[0] != rank + iter * 2 || id[numparticles - 1] != rank - iter || - q[numparticles - 1] != rank - iter * 2) { - LOG_ERROR("Data verification failed on rank %d for step %d! id[0]=%d/%d, " - "q[0]=%d/%d, id[end]=%d/%d, q[end]=%d/%d\n", - rank, iter, id[0], rank + iter, q[0], rank + iter * 2, id[numparticles - 1], - rank - iter, q[numparticles - 1], rank - iter * 2); - return FAIL; - } - #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer wait time: %.5e\n", t1 - t0); + LOG_WARNING("Transfer wait time: %.5e\n", t1 - t0); #endif for (int j = 0; j < 8; j++) { @@ -219,7 +230,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer close time: %.5e\n", t0 - t1); + LOG_WARNING("Transfer close time: %.5e\n", t0 - t1); #endif for (int i = 0; i < 8; i++) { @@ -233,7 +244,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Obj close time: %.5e\n", t1 - t0); + LOG_WARNING("Obj close time: %.5e\n", t1 - t0); #endif } // End for steps diff --git a/src/tests/misc/read_write_col_perf.c b/src/tests/misc/read_write_col_perf.c index cec102f72..9ec12ff42 100644 --- a/src/tests/misc/read_write_col_perf.c +++ b/src/tests/misc/read_write_col_perf.c @@ -31,7 +31,6 @@ #include #include #include "pdc.h" -#include "pdc_analysis.h" #include "pdc_timing.h" diff --git a/src/tests/misc/read_write_perf.c b/src/tests/misc/read_write_perf.c index 152140d5f..a50c97aa6 100644 --- a/src/tests/misc/read_write_perf.c +++ b/src/tests/misc/read_write_perf.c @@ -31,7 +31,6 @@ #include #include #include "pdc.h" -#include "pdc_analysis.h" #include "pdc_timing.h" diff --git a/src/tests/misc/vpicio.c b/src/tests/misc/vpicio.c index 3f9a28dfd..76624101f 100644 --- a/src/tests/misc/vpicio.c +++ b/src/tests/misc/vpicio.c @@ -21,7 +21,6 @@ * reproduce, distribute copies to the public, prepare derivative works, and * perform publicly and display publicly, and to permit other to do so. */ - #include #include #include @@ -33,8 +32,19 @@ #include #include "pdc.h" #include "pdc_timing.h" +//#include "/pscratch/sd/n/nlewi26/src/work_space/source/pdc/pi.h" + +#define dLEAP 2 +#define PRECISION_INIT 2 +#define PRECISION_HEX 20000000 -#define NPARTICLES 8388608 +#define WARP_SIZE 32 +#define WARPS_PER_BLOCK 2 +#define THREADS_PER_BLOCK (WARPS_PER_BLOCK * WARP_SIZE) +#define BLOCKS_NUM ((PRECISION_HEX / dLEAP / THREADS_PER_BLOCK) + 1) + +#define NUM_ITERATIONS 1 +#define NPARTICLES 8388608 double uniform_random_number() @@ -62,6 +72,7 @@ main(int argc, char **argv) double t0, t1; const char *obj_names[] = {"dX", "dY", "dZ", "Ux", "Uy", "Uz", "q", "i"}; char obj_name[64]; + const char *transformation_str = "raw"; pdcid_t transfer_requests[8]; @@ -78,10 +89,12 @@ main(int argc, char **argv) steps = atoi(argv[2]); if (argc >= 4) sleeptime = atoi(argv[3]); + if (argc >= 5) + transformation_str = argv[4]; if (rank == 0) - LOG_INFO("Writing %" PRIu64 " particles per rank for %d steps with %d sec sleep time.\n", - numparticles, steps, sleeptime); + LOG_WARNING("Writing %" PRIu64 " particles per rank for %d steps with %d sec sleep time.\n", + numparticles, steps, sleeptime); dims[0] = numparticles * size; @@ -99,17 +112,25 @@ main(int argc, char **argv) // create a pdc pdc_id = PDCinit("pdc"); + if (pdc_id == 0) { + LOG_ERROR("Failed to initialize PDC\n"); + return FAIL; + } // create a container property cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc_id); if (cont_prop <= 0) { - LOG_ERROR("Failed to create container property"); + LOG_ERROR("Failed to create container property\n"); return FAIL; } // create a container +#ifdef ENABLE_MPI cont_id = PDCcont_create_col("c1", cont_prop); +#else + cont_id = PDCcont_create("c1", cont_prop); +#endif if (cont_id <= 0) { - LOG_ERROR("Failed to create container"); + LOG_ERROR("Failed to create container\n"); return FAIL; } // create an object property @@ -152,7 +173,7 @@ main(int argc, char **argv) #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); if (rank == 0) - LOG_INFO("\n#Step %d\n", iter); + LOG_WARNING("\n#Step %d\n", iter); t0 = MPI_Wtime(); #endif @@ -168,13 +189,57 @@ main(int argc, char **argv) LOG_ERROR("Error getting an object id of %s from server\n", obj_name); return FAIL; } + if (!strcmp(transformation_str, "turbo") && obj_prop == obj_prop_int) { + if (rank == 0) + LOG_WARNING("Attaching turbo to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "turbo.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "compressed"); + } + else if (!strcmp(transformation_str, "zfp") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching zfp to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "compressed"); + } + else if (!strcmp(transformation_str, "zfp_gpu") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching zfp gpu to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_gpu.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "compressed"); + } + else if (!strcmp(transformation_str, "zfp_gpu_or_cpu") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching zfp gpu to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_gpu_or_cpu.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "compressed"); + } + else if (!strcmp(transformation_str, "sz") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching sz to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "sz.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "compressed"); + } + else if (!strcmp(transformation_str, "zfp_libsod") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching zfp_libsod to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_libsod.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "decompressed", "encrypted"); + if (rank == 0) + PDCtf_print_dg(dg_id, true); + } + else if (!strcmp(transformation_str, "custom") && obj_prop == obj_prop_float) { + if (rank == 0) + LOG_WARNING("Attaching custom to index: %d\n", obj_ids[i]); + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "custom.json"); + PDCtf_attach_to_obj(dg_id, obj_ids[i], "client", "server"); + } } #ifdef ENABLE_MPI MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Obj create time: %.5e\n", t1 - t0); + LOG_WARNING("Obj create time: %.5e\n", t1 - t0); #endif for (int i = 0; i < 8; i++) { @@ -190,7 +255,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer create time: %.5e\n", t0 - t1); + LOG_WARNING("Transfer create time: %.5e\n", t0 - t1); #endif #ifdef ENABLE_MPI @@ -206,15 +271,21 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer start time: %.5e\n", t1 - t0); + LOG_WARNING("Transfer start time: %.5e\n", t1 - t0); #endif // Emulate compute with sleep if (iter != steps - 1) { if (rank == 0) - LOG_INFO("Sleep start: %llu.00\n", sleeptime); + LOG_WARNING("Sleep start: %llu.00\n", sleeptime); + double loop_start = MPI_Wtime(); + // Call C function which launches kernel here sleep(sleeptime); + double loop_end = MPI_Wtime(); + if (rank == 0) { + LOG_WARNING("\nTotal time for %d iterations: %f s\n", NUM_ITERATIONS, loop_end - loop_start); + } if (rank == 0) - LOG_INFO("Sleep end: %llu.00\n", sleeptime); + LOG_WARNING("Sleep end: %llu.00\n", sleeptime); } #ifdef ENABLE_MPI @@ -231,7 +302,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer wait time: %.5e\n", t1 - t0); + LOG_WARNING("Transfer wait time: %.5e\n", t1 - t0); #endif for (int j = 0; j < 8; j++) { @@ -245,7 +316,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t0 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Transfer close time: %.5e\n", t0 - t1); + LOG_WARNING("Transfer close time: %.5e\n", t0 - t1); #endif for (int i = 0; i < 8; i++) { @@ -259,7 +330,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); t1 = MPI_Wtime(); if (rank == 0) - LOG_INFO("Obj close time: %.5e\n", t1 - t0); + LOG_WARNING("Obj close time: %.5e\n", t1 - t0); #endif } // End for steps @@ -275,7 +346,7 @@ main(int argc, char **argv) LOG_ERROR("Failed to close local region \n"); return FAIL; } - if (PDCobj_close(region_remote) != SUCCEED) { + if (PDCregion_close(region_remote) != SUCCEED) { LOG_ERROR("Failed to close remote region\n"); return FAIL; } diff --git a/src/tests/region/region_transfer.c b/src/tests/region/region_transfer.c index ca5578f2d..38e12d862 100644 --- a/src/tests/region/region_transfer.c +++ b/src/tests/region/region_transfer.c @@ -93,6 +93,9 @@ main(int argc, char **argv) "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_REGION_LOCAL) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); // create first object sprintf(obj_name1, "o1_%d", rank); diff --git a/src/tests/transform/pdc_transforms_lib.c b/src/tests/transform/pdc_transforms_lib.c deleted file mode 100644 index 4d3d08e97..000000000 --- a/src/tests/transform/pdc_transforms_lib.c +++ /dev/null @@ -1,695 +0,0 @@ -#include "pdc.h" -#include -#include - -/* - * >> pdc_public.h - * - * typedef enum { - * PDC_UNKNOWN = -1, * error * - * PDC_INT = 0, * integer types (identical to int32_t) * - * PDC_FLOAT = 1, * floating-point types * - * PDC_DOUBLE = 2, * double types * - * PDC_CHAR = 3, * character types * - * PDC_STRING = 4, * string types * - * PDC_BOOLEAN = 5, * boolean types * - * PDC_SHORT = 6, * short types * - * PDC_UINT = 7, * unsigned integer types (identical to uint32_t) * - * PDC_INT64 = 8, * 64-bit integer types * - * PDC_UINT64 = 9, * 64-bit unsigned integer types * - * PDC_INT16 = 10, * 16-bit integer types * - * PDC_INT8 = 11, * 8-bit integer types * - * PDC_UINT8 = 12, * 8-bit unsigned integer types * - * PDC_UINT16 = 13, * 16-bit unsigned integer types * - * PDC_LONG = 14, * long types * - * PDC_VOID_PTR = 15, * void pointer type * - * PDC_SIZE_T = 16, * size_t type * - * TYPE_COUNT = 17 * this is the number of var types and has to be the last * - * } pdc_c_var_type_t; - */ - -static int -_int_to_float(int *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_int_to_double(int *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_int_to_int64(int *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_int_to_int16(int *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_int_to_int8(int *dataIn, int8_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)(dataIn[k] & 0x0FF); - return 0; -} - -static int -_uint_to_float(uint32_t *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_uint_to_double(uint32_t *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_uint_to_int64(uint32_t *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_uint_to_int16(uint32_t *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_uint_to_int8(uint32_t *dataIn, int8_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)(dataIn[k] & 0x0FF); - return 0; -} - -static int -_int64_to_float(int64_t *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_int64_to_double(int64_t *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_int64_to_int(int64_t *dataIn, int *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int)dataIn[k]; - return 0; -} - -static int -_int64_to_int16(int64_t *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_int64_to_int8(int64_t *dataIn, int8_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)(dataIn[k] & 0x0FF); - return 0; -} - -static int -_uint64_to_float(uint64_t *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_uint64_to_double(uint64_t *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_uint64_to_int(uint64_t *dataIn, int *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int)dataIn[k]; - return 0; -} - -static int -_uint64_to_int16(uint64_t *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_uint64_to_int8(uint64_t *dataIn, int8_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)(dataIn[k] & 0x0FF); - return 0; -} - -static int -_int16_to_float(int16_t *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_int16_to_double(int16_t *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_int16_to_int64(int16_t *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_int16_to_uint64(int16_t *dataIn, uint64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (uint64_t)dataIn[k]; - return 0; -} - -static int -_int16_to_int(int16_t *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_int16_to_int8(int16_t *dataIn, int8_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)(dataIn[k] & 0x0FF); - return 0; -} - -static int -_int8_to_float(int8_t *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_int8_to_double(int8_t *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_int8_to_int64(int8_t *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_int8_to_uint64(int8_t *dataIn, uint64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (uint64_t)dataIn[k]; - return 0; -} - -static int -_int8_to_int(int8_t *dataIn, int *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_int8_to_int16(int8_t *dataIn, short *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (short)(dataIn[k] & 0x0FFFF); - return 0; -} - -static int -_float_to_int(float *dataIn, int *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int)dataIn[k]; - return 0; -} - -static int -_float_to_double(float *dataIn, double *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (double)dataIn[k]; - return 0; -} - -static int -_float_to_int64(float *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_float_to_int16(float *dataIn, int16_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int16_t)dataIn[k]; - return 0; -} - -static int -_float_to_int8(float *dataIn, int16_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)dataIn[k]; - return 0; -} - -static int -_double_to_int(double *dataIn, int *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int)dataIn[k]; - return 0; -} - -static int -_double_to_float(double *dataIn, float *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (float)dataIn[k]; - return 0; -} - -static int -_double_to_int64(double *dataIn, int64_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int64_t)dataIn[k]; - return 0; -} - -static int -_double_to_int16(double *dataIn, int16_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int16_t)dataIn[k]; - return 0; -} - -static int -_double_to_int8(double *dataIn, int16_t *dataOut, size_t elements) -{ - size_t k; - for (k = 0; k < elements; k++) - dataOut[k] = (int8_t)dataIn[k]; - return 0; -} - -/* - * The datatype conversion routines are really only useful - * when dealing with size changes or type class changes, e.g. - * integer to floating point or the converse. All other - * changes are effectively unsupported, e.g. integer - * to strings? - */ - -size_t -pdc_convert_datatype(void *dataIn, PDC_var_type_t srcType, int ndim, uint64_t *dims, void **dataOut, - PDC_var_type_t destType) -{ - int d; - size_t elements = dims[0]; - size_t storage_in_bytes; - for (d = 1; d < ndim; d++) - elements *= dims[d]; - elements /= 10; - if (destType >= 0) { - if ((destType == PDC_INT) || (destType == PDC_UINT) || (destType == PDC_FLOAT)) - storage_in_bytes = elements * sizeof(int); - else if ((destType == PDC_DOUBLE) || (destType == PDC_INT64) || (destType == PDC_UINT64)) - storage_in_bytes = elements * sizeof(double); - else if (destType == PDC_INT16) - storage_in_bytes = elements * sizeof(short); - else if (destType == PDC_INT8) - storage_in_bytes = elements; - else - storage_in_bytes = elements; - } - LOG_INFO("pdc_convert_datatype: elements = %ld\n", elements); - - switch (srcType) { - case PDC_INT: { - if (destType == PDC_FLOAT) - _int_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _int_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _int_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _int_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _int_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_FLOAT: { - if (destType == PDC_INT) - _float_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _float_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _float_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _float_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _float_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_DOUBLE: { - if (destType == PDC_INT) - _double_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_FLOAT) - _double_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _double_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _double_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _double_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_UINT: { - if (destType == PDC_FLOAT) - _uint_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _uint_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _uint_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _uint_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _uint_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_INT64: { - if (destType == PDC_INT) - _int64_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_FLOAT) - _int64_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _int64_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _int64_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _int64_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_UINT64: { - if (destType == PDC_INT) - _uint64_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_FLOAT) - _uint64_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _uint64_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _uint64_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _uint64_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_INT16: { - if (destType == PDC_INT) - _int16_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_FLOAT) - _int16_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _int16_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _int16_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_UINT64) - _int16_to_uint64(dataIn, *dataOut, elements); - else if (destType == PDC_INT8) - _int16_to_int8(dataIn, *dataOut, elements); - else - return -1; - } break; - case PDC_INT8: { - if (destType == PDC_INT) - _int8_to_int(dataIn, *dataOut, elements); - else if (destType == PDC_FLOAT) - _int8_to_float(dataIn, *dataOut, elements); - else if (destType == PDC_DOUBLE) - _int8_to_double(dataIn, *dataOut, elements); - else if (destType == PDC_INT16) - _int8_to_int16(dataIn, *dataOut, elements); - else if (destType == PDC_INT64) - _int8_to_int64(dataIn, *dataOut, elements); - else if (destType == PDC_UINT64) - _int8_to_uint64(dataIn, *dataOut, elements); - else - return -1; - } break; - default: - return -1; - } - return storage_in_bytes; -} - -int -pdc_transform_increment(void *dataIn, PDC_var_type_t srcType, int ndim, uint64_t *dims, void **dataOut, - PDC_var_type_t destType) -{ - int i; - int64_t nval, nbytes; - void * destBuff; - size_t typesize = 1; - if (srcType >= 0) { - if ((srcType == PDC_INT) || (srcType == PDC_UINT) || (srcType == PDC_FLOAT)) - typesize = sizeof(int); - else if ((srcType == PDC_DOUBLE) || (srcType == PDC_INT64) || (srcType == PDC_UINT64)) - typesize = sizeof(double); - else if (srcType == PDC_INT16) - typesize = sizeof(short); - } - - LOG_INFO("\n[TRANSFORM] Entering pdc_transform_increment\n"); - - nval = 1; - for (i = 0; i < ndim; i++) { - nval *= dims[i]; - } - - nbytes = nval * typesize; - destBuff = malloc(nbytes); - memcpy(destBuff, dataIn, nbytes); - - switch (srcType) { - case PDC_INT: - for (i = 0; i < nval; i++) { - ((int *)(destBuff))[i]++; - ; - } - break; - case PDC_FLOAT: - for (i = 0; i < nval; i++) { - ((float *)(destBuff))[i]++; - ; - } - break; - default: - LOG_ERROR("\n[TRANSFORM] Unable to increment values\n"); - return -1; - break; - } - - *dataOut = destBuff; - - LOG_INFO("\n[TRANSFORM] %ld values successfully incremented\n", nval); - return nval; -} - -/* - * https://github.com/Blosc/c-blosc/blob/master/blosc/blosc.h - */ -size_t -pdc_transform_compress(void *dataIn, PDC_var_type_t srcType, int ndim, uint64_t *dims, void **dataOut, - PDC_var_type_t destType) -{ - int clevel = 9; - int doshuffle = BLOSC_BITSHUFFLE; - int nval, i, typesize; - int64_t destsize, nbytes, csize; - void * destBuff; - - if ((srcType == PDC_INT) || (srcType == PDC_UINT) || (srcType == PDC_FLOAT)) - typesize = sizeof(int); - else if ((srcType == PDC_DOUBLE) || (srcType == PDC_INT64) || (srcType == PDC_UINT64)) - typesize = sizeof(double); - else if (srcType == PDC_INT16) - typesize = sizeof(short); - else if (srcType == PDC_INT8) - typesize = 1; - - LOG_INFO("\n[TRANSFORM] Entering pdc_transform_compress\n"); - nval = 1; - for (i = 0; i < ndim; i++) - nval *= dims[i]; - - nbytes = nval * typesize; - LOG_INFO("[TRANSFORM] Src Buffer size : %d * %d = %ld\n", nval, typesize, nbytes); - destsize = nbytes + BLOSC_MAX_OVERHEAD; - LOG_INFO("[TRANSFORM] Dest Buffer size : %ld + %d = %ld\n", nbytes, BLOSC_MAX_OVERHEAD, destsize); - destBuff = malloc(destsize); - - csize = blosc_compress(clevel, doshuffle, typesize, nbytes, dataIn, destBuff, destsize); - - if (csize < 0) - LOG_ERROR("[TRANSFORM] Error while compressing data (errcode: %ld)\n", csize); - if (csize == 0) - LOG_ERROR("[TRANSFORM] Unable to compress data (errcode: %ld)\n", csize); - - if (csize > 0) { - LOG_INFO("[TRANSFORM] Data sucessfully compressed from %ld B to %ld B\n", nbytes, csize); - } - *dataOut = destBuff; - return csize; -} - -size_t -pdc_transform_decompress(void *dataIn, PDC_var_type_t srcType, int ndim, uint64_t *dims, void **dataOut, - PDC_var_type_t destType) -{ - int i; - size_t typesize, destsize, dsize; - void * destBuff = *dataOut; - - LOG_INFO("\n[TRANSFORM] Entering pdc_transform_decompress\n"); - - if ((destType == PDC_INT) || (destType == PDC_UINT) || (destType == PDC_FLOAT)) - typesize = sizeof(int); - else if ((destType == PDC_DOUBLE) || (destType == PDC_INT64) || (destType == PDC_UINT64)) - typesize = sizeof(double); - else if (destType == PDC_INT16) - typesize = sizeof(short); - else if (destType == PDC_INT8) - typesize = 1; - - /* destsize is the byte length of destBuff */ - destsize = typesize; - for (i = 0; i < ndim; i++) - destsize *= dims[i]; - - dsize = (size_t)blosc_decompress(dataIn, destBuff, destsize); - - if (dsize <= 0) - LOG_ERROR("[TRANSFORM] Error while decompressing data (errcode: %zu)\n", dsize); - else - LOG_INFO("[TRANSFORM] Data sucessfully decompressed\n"); - return dsize; -} diff --git a/src/tests/transformation/all_compression_transform.c b/src/tests/transformation/all_compression_transform.c new file mode 100644 index 000000000..5b178ffd9 --- /dev/null +++ b/src/tests/transformation/all_compression_transform.c @@ -0,0 +1,241 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 256 +#define OBJ_NUM 10 + +#define BATCH_REQUESTS 1 +#define INDIVIDUAL_REQUESTS 0 + +#define TRANSFORM_GRAPH_PATH TF_GRAPHS_DIR "zfp.json" + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t *obj; + pdcid_t *dg_ids; + char cont_name[128], obj_name[128]; + pdcid_t *transfer_request; + + int rank = 0, size = 1, i, j; + int ret_value = 0; + int **data, **data_read; + int start_method = BATCH_REQUESTS; + int wait_method = BATCH_REQUESTS; + + uint64_t offset[1], offset_length[1]; + uint64_t dims[1]; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + if (argc >= 2) + start_method = atoi(argv[1]); + if (argc >= 3) + wait_method = atoi(argv[2]); + if (!rank) { + LOG_INFO("Start method: %s\n", (start_method) ? "batch" : "individual"); + LOG_INFO("Wait method: %s\n", (start_method) ? "batch" : "individual"); + } + + data = (int **)malloc(sizeof(int *) * OBJ_NUM); + data_read = (int **)malloc(sizeof(int *) * OBJ_NUM); + data[0] = (int *)malloc(sizeof(int) * BUF_LEN * OBJ_NUM); + data_read[0] = (int *)malloc(sizeof(int) * BUF_LEN * OBJ_NUM); + + for (i = 1; i < OBJ_NUM; ++i) { + data[i] = data[i - 1] + BUF_LEN; + data_read[i] = data_read[i - 1] + BUF_LEN; + } + + dims[0] = BUF_LEN; + + // create a pdc + pdc = PDCinit("pdc"); + LOG_INFO("Created a new pdc\n"); + + // create a container property + cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc); + if (cont_prop > 0) + LOG_INFO("Created container property\n"); + else + TGOTO_ERROR(FAIL, "Failed to create container property"); + + // create a container + sprintf(cont_name, "c%d", rank); + cont = PDCcont_create(cont_name, cont_prop); + if (cont > 0) + LOG_INFO("Created container c1\n"); + else + TGOTO_ERROR(TFAIL, "Failed to create container"); + // create an object property + obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc); + if (obj_prop > 0) + LOG_INFO("Created an object property\n"); + else { + LOG_ERROR("Failed to create object property\n"); + ret_value = 1; + } + + ret = PDCprop_set_obj_type(obj_prop, PDC_INT); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to set obj type\n"); + PDCprop_set_obj_dims(obj_prop, 1, dims); + PDCprop_set_obj_user_id(obj_prop, getuid()); + PDCprop_set_obj_time_step(obj_prop, 0); + PDCprop_set_obj_app_name(obj_prop, "DataServerTest"); + PDCprop_set_obj_tags(obj_prop, "tag0=1"); + + // create many objects + obj = (pdcid_t *)malloc(sizeof(pdcid_t) * OBJ_NUM); + dg_ids = (pdcid_t *)malloc(sizeof(pdcid_t) * OBJ_NUM); + for (i = 0; i < OBJ_NUM; ++i) { + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to set obj type"); + + sprintf(obj_name, "o%d_%d", i, rank); + obj[i] = PDCobj_create(cont, obj_name, obj_prop); + if (obj[i] > 0) + LOG_INFO("Created an object o1\n"); + else + TGOTO_ERROR(TFAIL, "Failed to create object"); + } + + offset[0] = 0; + offset_length[0] = BUF_LEN; + reg = PDCregion_create(1, offset, offset_length); + if (reg > 0) + LOG_INFO("Create local region\n"); + else + TGOTO_ERROR(TFAIL, "Failed to create region\n"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + reg_global = PDCregion_create(1, offset, offset_length); + if (reg_global > 0) + LOG_INFO("Created a global region\n"); + else + TGOTO_ERROR(TFAIL, "Failed to create region"); + + for (j = 0; j < OBJ_NUM; ++j) { + for (i = 0; i < BUF_LEN; ++i) + data[j][i] = i; + } + transfer_request = (pdcid_t *)malloc(sizeof(pdcid_t) * OBJ_NUM); + + // Place a transfer request for every objects + for (i = 0; i < OBJ_NUM; ++i) { + dg_ids[i] = PDCtf_dg_json_create(TRANSFORM_GRAPH_PATH); + PDCtf_attach_to_region(dg_ids[i], obj[i], reg_global, "decompressed", "compressed"); + transfer_request[i] = PDCregion_transfer_create(data[i], PDC_WRITE, obj[i], reg, reg_global); + } + if (start_method) { + ret = PDCregion_transfer_start_all(transfer_request, OBJ_NUM); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer start"); + } + else { + for (i = 0; i < OBJ_NUM; ++i) { + ret = PDCregion_transfer_start(transfer_request[i]); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer start"); + } + } + if (wait_method == 1) { + ret = PDCregion_transfer_wait_all(transfer_request, OBJ_NUM); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer wait"); + } + else if (wait_method == 0) { + pdcid_t *transfer_request_all = (pdcid_t *)malloc(sizeof(pdcid_t) * OBJ_NUM); + int request_size = 0; + for (i = 0; i < OBJ_NUM; i += 2) { + transfer_request_all[request_size] = transfer_request[i]; + request_size++; + } + ret = PDCregion_transfer_wait_all(transfer_request_all, request_size); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer wait"); + request_size = 0; + for (i = 1; i < OBJ_NUM; i += 2) { + transfer_request_all[request_size] = transfer_request[i]; + request_size++; + } + ret = PDCregion_transfer_wait_all(transfer_request_all, request_size); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer wait"); + free(transfer_request_all); + } + for (i = 0; i < OBJ_NUM; ++i) { + PDCtf_close_dg(dg_ids[i]); + ret = PDCregion_transfer_close(transfer_request[i]); + if (ret != SUCCEED) + TGOTO_ERROR(TFAIL, "Failed to region transfer close"); + } + if (PDCregion_close(reg) < 0) + TGOTO_ERROR(TFAIL, "Failed to close local region"); + else + LOG_INFO("Successfully closed local region\n"); + + if (PDCregion_close(reg_global) < 0) + TGOTO_ERROR(TFAIL, "Failed to close global region\n"); + else + LOG_INFO("Successfully closed global region\n"); + +done: + free(data[0]); + free(data_read[0]); + free(data); + free(data_read); + free(obj); + free(transfer_request); + // close pdc + if (PDCclose(pdc) < 0) + TGOTO_ERROR(TFAIL, "Failed to close PDC"); + +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/attach_obj_compression_transform.c b/src/tests/transformation/attach_obj_compression_transform.c new file mode 100644 index 000000000..f140da462 --- /dev/null +++ b/src/tests/transformation/attach_obj_compression_transform.c @@ -0,0 +1,186 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 1D region transfer write and then a single region transfer read. + * Attached a compression/decompression transformation to the region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = TSUCCEED; + + uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t dims[1]; + local_offset[0] = 0; + offset[0] = 0; + offset[1] = 2; + offset[2] = 5; + offset_length[0] = BUF_LEN; + offset_length[1] = 3; + offset_length[2] = 5; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = PDC_SIZE_UNLIMITED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 1, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + PDCtf_attach_to_obj(dg_id, obj1, "decompressed", "compressed"); + + // write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT((reg = PDCregion_create(1, local_offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + memset(data_read, 0, sizeof(int) * BUF_LEN); + + // read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_encryption_transform.c b/src/tests/transformation/compression_encryption_transform.c new file mode 100644 index 000000000..804076b41 --- /dev/null +++ b/src/tests/transformation/compression_encryption_transform.c @@ -0,0 +1,188 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 1D region transfer write and then a single region transfer read. + * Attached a compression/decompression encyprtion/decryption transformation to the region before the region + * write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = TSUCCEED; + + uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t dims[1]; + local_offset[0] = 0; + offset[0] = 0; + offset[1] = 2; + offset[2] = 5; + offset_length[0] = BUF_LEN; + offset_length[1] = 3; + offset_length[2] = 5; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = PDC_SIZE_UNLIMITED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + + TASSERT(PDCprop_set_obj_dims(obj_prop, 1, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_libsod.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "encrypted"); + + // write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT((reg = PDCregion_create(1, local_offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + memset(data_read, 0, sizeof(int) * BUF_LEN); + + // read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_encryption_transform_2D.c b/src/tests/transformation/compression_encryption_transform_2D.c new file mode 100644 index 000000000..5f9dd2764 --- /dev/null +++ b/src/tests/transformation/compression_encryption_transform_2D.c @@ -0,0 +1,194 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 2D region transfer write and then a single region transfer read. + * Attached a compression/decompression encyprtion/decryption transformation to the region before the region + * write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = 0; + + uint64_t offset[3], offset_length[3]; + uint64_t dims[2]; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = BUF_LEN / 4; + dims[1] = 4; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d_2d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 2, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to (PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d_2d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d_2d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset_length[0] = BUF_LEN / 4; + offset_length[1] = 4; + TASSERT((reg_global = PDCregion_create(2, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) { + data[i] = i; + } + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_libsod.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "encrypted"); + + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset_length[0] = BUF_LEN / 4; + offset_length[1] = 4; + TASSERT((reg_global = PDCregion_create(2, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(TFAIL, "Wrong value %d!=%d\n", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_encryption_transform_3D.c b/src/tests/transformation/compression_encryption_transform_3D.c new file mode 100644 index 000000000..729ad862a --- /dev/null +++ b/src/tests/transformation/compression_encryption_transform_3D.c @@ -0,0 +1,193 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 3D region transfer write and then a single region transfer read. + * Attached a compression/decompression encyprtion/decryption transformation to the region before the region + * write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 256 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1; + char cont_name[128], obj_name1[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = 0; + + uint64_t offset[3], offset_length[3]; + uint64_t dims[3]; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = BUF_LEN / 16; + dims[1] = 4; + dims[2] = 4; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 3, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset[2] = 0; + offset_length[0] = BUF_LEN / 16; + offset_length[1] = 4; + offset_length[2] = 4; + TASSERT((reg_global = PDCregion_create(3, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) { + data[i] = i; + } + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_libsod.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "encrypted"); + + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset[2] = 0; + offset_length[0] = BUF_LEN / 16; + offset_length[1] = 4; + offset_length[2] = 4; + TASSERT((reg_global = PDCregion_create(3, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(TFAIL, "Wrong value %d!=%d\n", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_transform.c b/src/tests/transformation/compression_transform.c new file mode 100644 index 000000000..a79d2b6c8 --- /dev/null +++ b/src/tests/transformation/compression_transform.c @@ -0,0 +1,186 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 1D region transfer write and then a single region transfer read. + * Attached a compression/decompression transformation to the region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = TSUCCEED; + + uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t dims[1]; + local_offset[0] = 0; + offset[0] = 0; + offset[1] = 2; + offset[2] = 5; + offset_length[0] = BUF_LEN; + offset_length[1] = 3; + offset_length[2] = 5; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = PDC_SIZE_UNLIMITED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 1, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_REGION_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "compressed"); + + // write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT((reg = PDCregion_create(1, local_offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + memset(data_read, 0, sizeof(int) * BUF_LEN); + + // read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_transform_2D.c b/src/tests/transformation/compression_transform_2D.c new file mode 100644 index 000000000..c87be4df9 --- /dev/null +++ b/src/tests/transformation/compression_transform_2D.c @@ -0,0 +1,193 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 2D region transfer write and then a single region transfer read. + * Attached a compression/decompression transformation to the region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = 0; + + uint64_t offset[3], offset_length[3]; + uint64_t dims[2]; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = BUF_LEN / 4; + dims[1] = 4; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d_2d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 2, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to (PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d_2d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d_2d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset_length[0] = BUF_LEN / 4; + offset_length[1] = 4; + TASSERT((reg_global = PDCregion_create(2, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) { + data[i] = i; + } + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "compressed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset_length[0] = BUF_LEN / 4; + offset_length[1] = 4; + TASSERT((reg_global = PDCregion_create(2, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(TFAIL, "Wrong value %d!=%d\n", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_transform_3D.c b/src/tests/transformation/compression_transform_3D.c new file mode 100644 index 000000000..c73a5c259 --- /dev/null +++ b/src/tests/transformation/compression_transform_3D.c @@ -0,0 +1,192 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 3D region transfer write and then a single region transfer read. + * Attached a compression/decompression transformation to the region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 256 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1; + char cont_name[128], obj_name1[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = 0; + + uint64_t offset[3], offset_length[3]; + uint64_t dims[3]; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = BUF_LEN / 16; + dims[1] = 4; + dims[2] = 4; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 3, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset[2] = 0; + offset_length[0] = BUF_LEN / 16; + offset_length[1] = 4; + offset_length[2] = 4; + TASSERT((reg_global = PDCregion_create(3, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) { + data[i] = i; + } + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "compressed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + offset[0] = 0; + offset[1] = 0; + offset[2] = 0; + offset_length[0] = BUF_LEN / 16; + offset_length[1] = 4; + offset_length[2] = 4; + TASSERT((reg_global = PDCregion_create(3, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(TFAIL, "Wrong value %d!=%d\n", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/compression_transform_gpu.c b/src/tests/transformation/compression_transform_gpu.c new file mode 100644 index 000000000..b8e49ac3f --- /dev/null +++ b/src/tests/transformation/compression_transform_gpu.c @@ -0,0 +1,186 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a single 1D region transfer write and then a single region transfer read. + * Attached a compression/decompression transformation to the region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = TSUCCEED; + + uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t dims[1]; + local_offset[0] = 0; + offset[0] = 0; + offset[1] = 2; + offset[2] = 5; + offset_length[0] = BUF_LEN; + offset_length[1] = 3; + offset_length[2] = 5; + + double *data = (double *)malloc(sizeof(double) * BUF_LEN); + double *data_read = (double *)malloc(sizeof(double) * BUF_LEN); + dims[0] = PDC_SIZE_UNLIMITED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_DOUBLE) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 1, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, offset, offset_length)) != 0, "Call to PDCregion_create succeeded", + "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + for (i = 0; i < BUF_LEN; ++i) + data[i] = 900.120937 + i; + + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "compression_gpu.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "decompressed", "compressed"); + + // write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + + TASSERT((reg = PDCregion_create(1, local_offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + // read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + LOG_JUST_PRINT("%lf ", data_read[i]); + if (i % 10 == 0 && i != 0) { + LOG_JUST_PRINT("\n"); + } + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tests/transformation/multi_region_transform.c b/src/tests/transformation/multi_region_transform.c new file mode 100644 index 000000000..d85968a72 --- /dev/null +++ b/src/tests/transformation/multi_region_transform.c @@ -0,0 +1,251 @@ +/* + * Copyright Notice for + * Proactive Data Containers (PDC) Software Library and Utilities + * ----------------------------------------------------------------------------- + + *** Copyright Notice *** + + * Proactive Data Containers (PDC) Copyright (c) 2017, The Regents of the + * University of California, through Lawrence Berkeley National Laboratory, + * UChicago Argonne, LLC, operator of Argonne National Laboratory, and The HDF + * Group (subject to receipt of any required approvals from the U.S. Dept. of + * Energy). All rights reserved. + + * If you have questions about your rights to use or distribute this software, + * please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. + + * NOTICE. This Software was developed under funding from the U.S. Department of + * Energy and the U.S. Government consequently retains certain rights. As such, the + * U.S. Government has been granted for itself and others acting on its behalf a + * paid-up, nonexclusive, irrevocable, worldwide license in the Software to + * reproduce, distribute copies to the public, prepare derivative works, and + * perform publicly and display publicly, and to permit other to do so. + */ + +/** + * Test Description + * ----------------------------------------------------------------------------- + * + * Performs a 3 1D region transfer writes and 3 corresponding reads. + * No region transformation to the first region before the region write/read. + * Attached a compression/decompression transformation to the 2nd region before the region write/read. + * Attached a encryption/decryption transformation to the 2nd region before the region write/read. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "pdc.h" +#include "test_helper.h" + +#define BUF_LEN 128 + +int +main(int argc, char **argv) +{ + pdcid_t pdc, cont_prop, cont, obj_prop, reg, reg_global; + perr_t ret; + pdcid_t obj1, obj2; + char cont_name[128], obj_name1[128], obj_name2[128]; + pdcid_t transfer_request; + + int rank = 0, size = 1, i; + int ret_value = TSUCCEED; + + uint64_t offset[3], offset_length[3], local_offset[1]; + uint64_t dims[1]; + local_offset[0] = 0; + offset[0] = 0; + offset[1] = 2; + offset[2] = 5; + offset_length[0] = BUF_LEN; + offset_length[1] = 3; + offset_length[2] = 5; + + int *data = (int *)malloc(sizeof(int) * BUF_LEN); + int *data_read = (int *)malloc(sizeof(int) * BUF_LEN); + dims[0] = PDC_SIZE_UNLIMITED; + +#ifdef ENABLE_MPI + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + + // create a pdc + TASSERT((pdc = PDCinit("pdc")) != 0, "Call to PDCinit succeeded", "Call to PDCinit failed"); + // create a container property + TASSERT((cont_prop = PDCprop_create(PDC_CONT_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + // create a container + sprintf(cont_name, "c%d", rank); + TASSERT((cont = PDCcont_create(cont_name, cont_prop)) != 0, "Call to PDCcont_create succeeded", + "Call to PDCcont_create failed"); + // create an object property + TASSERT((obj_prop = PDCprop_create(PDC_OBJ_CREATE, pdc)) != 0, "Call to PDCprop_create succeeded", + "Call to PDCprop_create failed"); + TASSERT(PDCprop_set_obj_type(obj_prop, PDC_INT) >= 0, "Call to PDCprop_set_obj_type succeeded", + "Call to PDCprop_set_obj_type failed"); + TASSERT(PDCprop_set_obj_dims(obj_prop, 1, dims) >= 0, "Call to PDCprop_set_obj_dims succeeded", + "Call to PDCprop_set_obj_dims failed"); + TASSERT(PDCprop_set_obj_user_id(obj_prop, getuid()) >= 0, "Call to PDCprop_set_obj_user_id succeeded", + "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_time_step(obj_prop, 0) >= 0, "Call to PDCprop_set_obj_time_step succeeded", + "Call to PDCprop_set_obj_time_step failed"); + TASSERT(PDCprop_set_obj_app_name(obj_prop, "DataServerTest") >= 0, + "Call to PDCprop_set_obj_user_id succeeded", "Call to PDCprop_set_obj_user_id failed"); + TASSERT(PDCprop_set_obj_tags(obj_prop, "tag0=1") >= 0, "Call to PDCprop_set_obj_tags succeeded", + "Call to PDCprop_set_obj_tags failed"); + TASSERT(PDCprop_set_obj_transfer_region_type(obj_prop, PDC_OBJ_STATIC) >= 0, + "Call to PDCprop_set_obj_transfer_region_type succeeded", + "Call to PDCprop_set_obj_transfer_region_type failed"); + + // create first object + sprintf(obj_name1, "o1_%d", rank); + TASSERT((obj1 = PDCobj_create(cont, obj_name1, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + // create second object + sprintf(obj_name2, "o2_%d", rank); + TASSERT((obj2 = PDCobj_create(cont, obj_name2, obj_prop)) != 0, "Call to PDCobj_create succeeded", + "Call to PDCobj_create failed"); + + offset[0] = 0; + offset_length[0] = BUF_LEN; + TASSERT((reg = PDCregion_create(1, local_offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + // Write to data buffer + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + // Write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + memset(data_read, 0, sizeof(int) * BUF_LEN); + // Read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // Redo offsets + offset[0] = BUF_LEN; + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + // attach graph + pdcid_t dg_id = PDCtf_dg_json_create(TF_GRAPHS_DIR "zfp_libsod.json"); + PDCtf_attach_to_region(dg_id, obj1, reg_global, "client", "compressed"); + + // Write to data buffer + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + // Write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + memset(data_read, 0, sizeof(int) * BUF_LEN); + // Read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // Redo offsets + offset[0] = BUF_LEN * 2; + TASSERT((reg_global = PDCregion_create(1, offset, offset_length)) != 0, + "Call to PDCregion_create succeeded", "Call to PDCregion_create failed"); + + // attach graph + PDCtf_attach_to_region(dg_id, obj1, reg_global, "client", "encrypted"); + + // Write to data buffer + for (i = 0; i < BUF_LEN; ++i) + data[i] = i; + // Write transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data, PDC_WRITE, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + memset(data_read, 0, sizeof(int) * BUF_LEN); + // Read transfer request + TASSERT((transfer_request = PDCregion_transfer_create(data_read, PDC_READ, obj1, reg, reg_global)) != 0, + "Call to PDCregion_transfer_create succeeded", "Call to PDCregion_transfer_create failed"); + TASSERT(PDCregion_transfer_start(transfer_request) >= 0, "Call to PDCregion_transfer_start succeeded", + "Call to PDCregion_transfer_start failed"); + TASSERT(PDCregion_transfer_wait(transfer_request) >= 0, "Call to PDCregion_transfer_wait succeeded", + "Call to PDCregion_transfer_wait failed"); + TASSERT(PDCregion_transfer_close(transfer_request) >= 0, "Call to PDCregion_transfer_close succeeded", + "Call to PDCregion_transfer_close failed"); + // Check if data written previously has been correctly read. + for (i = 0; i < BUF_LEN; ++i) { + if (data_read[i] != i) + TGOTO_ERROR(FAIL, "Wrong value at index %d!=%d", data_read[i], i); + } + + // close tf graph + TASSERT(PDCtf_close_dg(dg_id) >= 0, "Call to PDCtf_close_dg succeeded", "Call to PDCtf_close_dg failed"); + // close regions + TASSERT(PDCregion_close(reg) >= 0, "Call to PDCregion_close succeeded", "Call to PDCregion_close failed"); + TASSERT(PDCregion_close(reg_global) >= 0, "Call to PDCregion_close succeeded", + "Call to PDCregion_close failed"); + // close object + TASSERT(PDCobj_close(obj1) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + TASSERT(PDCobj_close(obj2) >= 0, "Call to PDCobj_close succeeded", "Call to PDCobj_close failed"); + // close a container + TASSERT(PDCcont_close(cont) >= 0, "Call to PDCcont_close succeeded", "Call to PDCcont_close failed"); + // close a object property + TASSERT(PDCprop_close(obj_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close a container property + TASSERT(PDCprop_close(cont_prop) >= 0, "Call to PDCprop_close succeeded", "Call to PDCprop_close failed"); + // close pdc + TASSERT(PDCclose(pdc) >= 0, "Call to PDCclose succeeded", "Call to PDCclose failed"); + + free(data); + free(data_read); + +done: +#ifdef ENABLE_MPI + MPI_Finalize(); +#endif + return ret_value; +} diff --git a/src/tools/.gitignore b/src/tools/.gitignore new file mode 100644 index 000000000..3b969bed8 --- /dev/null +++ b/src/tools/.gitignore @@ -0,0 +1,12 @@ +CMakeCache.txt +CMakeFiles +Makefile +pdc_access_eqsim +pdc_export +pdc_import +pdc_import_eqsim +pdc_ls +pdc_query_eqsim +cmake_install.cmake +hdf5_access_eqsim +libcjson.a diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index b68348099..2c4eb5fd8 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -14,6 +14,11 @@ set(TOOLS_EXT_LIB "") # ************************************************* # * MERCURY # ************************************************* +if(NOT CMAKE_PREFIX_PATH) + set(CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX}) +endif() +# MERCURY +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} $ENV{MERCURY_DIR}) find_package(MERCURY REQUIRED) if(MERCURY_FOUND) set(TOOLS_EXT_INCLUDE ${MERCURY_INCLUDE_DIRS} @@ -21,6 +26,10 @@ if(MERCURY_FOUND) ) set(TOOLS_EXT_LIB ${MERCURY_LIBRARIES} ${TOOLS_EXT_LIB}) endif() +include_directories( + /mnt/fast/nlewis/workspace/install/mercury/include + ${TOOLS_EXT_INCLUDE} # your other includes +) #HDF5 find_package(HDF5 MODULE) @@ -81,45 +90,64 @@ endif() # endif() # endif() +set(PDC_SOURCE_DIR /mnt/fast/nlewis/workspace/source/pdc) + include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${PDC_INCLUDES_BUILD_TIME} ${PROJECT_BINARY_DIR} ${PDC_SOURCE_DIR} - ${PDC_SOURCE_DIR}/src/client_api/include + ${PDC_SOURCE_DIR}/src/api/include + ${PDC_SOURCE_DIR}/src/commons/include + ${PDC_SOURCE_DIR}/src/commons/generic/include + ${PDC_SOURCE_DIR}/src/api/pdc_obj/include + ${PDC_SOURCE_DIR}/src/commons/logging/include + ${PDC_SOURCE_DIR}/src/server/include + ${PDC_SOURCE_DIR}/src/commons/utils/include + ${PDC_SOURCE_DIR}/src/api/pdc_query/include + ${PDC_SOURCE_DIR}/src/api/pdc_tf/include + ${PDC_SOURCE_DIR}/src/server/transform/include + ${PDC_SOURCE_DIR}/src/server/pdc_server_region/include + ${PDC_SOURCE_DIR}/src/commons/collections/include + ${PDC_SOURCE_DIR}/src/commons/index/dart/include + ${PDC_SOURCE_DIR}/src/api/pdc_region/include + ${PDC_SOURCE_DIR}/src/commons/profiling/include + ${PDC_SOURCE_DIR}/src/commons/collections/libhl/include + ${PDC_SOURCE_DIR}/src/commons/file/include + ${PDC_SOURCE_DIR}/build $ENV{HOME}/Sandbox/c-blosc/blosc $ENV{HOME}/include ${TOOLS_EXT_INCLUDE} ) - set(PROGRAMS - pdc_import - pdc_export + #pdc_import + #pdc_export pdc_ls - pdc_import_eqsim - pdc_access_eqsim - pdc_query_eqsim - hdf5_access_eqsim - ) + #pdc_import_eqsim + #pdc_access_eqsim + #pdc_query_eqsim + #hdf5_access_eqsim +) add_library(cjson cjson/cJSON.c) +link_directories(${PDC_SOURCE_DIR}/build/bin) + foreach(program ${PROGRAMS}) add_executable(${program} ${program}.c) - target_link_libraries(${program} pdc cjson ${TOOLS_EXT_LIB}) + target_link_libraries(${program} pdc pdc_commons mercury m cjson ${TOOLS_EXT_LIB}) endforeach(program) - -add_executable(metadata_json_loader - metadata_json_loader.c - meta_json/metadata_json_processor.h - meta_json/metadata_json_printer.c - meta_json/metadata_json_printer.h - meta_json/metadata_json_importer.c - meta_json/metadata_json_importer.h - fs/fs_ops.c - fs/fs_ops.h) -target_link_libraries(metadata_json_loader ${PDC_EXT_LIB_DEPENDENCIES} pdc cjson ${TOOLS_EXT_LIB}) -target_include_directories(metadata_json_loader PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES}) +#add_executable(metadata_json_loader +# metadata_json_loader.c +# meta_json/metadata_json_processor.h +# meta_json/metadata_json_printer.c +# meta_json/metadata_json_printer.h +# meta_json/metadata_json_importer.c +# meta_json/metadata_json_importer.h +# fs/fs_ops.c +# fs/fs_ops.h) +#target_link_libraries(metadata_json_loader ${PDC_EXT_LIB_DEPENDENCIES} pdc cjson ${TOOLS_EXT_LIB}) +#target_include_directories(metadata_json_loader PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES}) diff --git a/src/tools/pdc_ls.c b/src/tools/pdc_ls.c index c9fae337e..8a3eda17c 100644 --- a/src/tools/pdc_ls.c +++ b/src/tools/pdc_ls.c @@ -5,11 +5,13 @@ #include #include #include +#include #include #include "pdc.h" #include "pdc_client_server_common.h" #include "../src/server/include/pdc_server_metadata.h" +#include "/mnt/fast/nlewis/workspace/source/pdc/src/server/transform/include/pdc_tf_server.h" #include "cjson/cJSON.h" const char *avail_args[] = {"-n", "-i", "-json", "-ln", "-li", "-s"}; @@ -60,6 +62,12 @@ typedef struct FileNameNode { struct FileNameNode *next; } FileNameNode; +typedef struct pdc_tf_obj_id_to_dg_t { + pdcid_t obj_id; + struct pdc_tf_obj_t pdc_tf_obj; + pdc_dg_t * dg; +} pdc_tf_obj_id_to_dg_t; + typedef struct ArrayList { int length; int capacity; @@ -128,9 +136,8 @@ int main(int argc, char *argv[]) { if (argc == 1) { - LOG_JUST_PRINT( - "Usage: ./pdc_ls pdc_checkpoint_directory/file [-n obj_name] [-i obj_id] [-json json_fname] " - "[-ln (list all names)] [-ls (list all ids)] [-s (summary)]\n"); + printf("Usage: ./pdc_ls pdc_checkpoint_directory/file [-n obj_name] [-i obj_id] [-json json_fname] " + "[-ln (list all names)] [-ls (list all ids)] [-s (summary)]\n"); return 0; } else { @@ -364,6 +371,18 @@ do_transfer_request_metadata(int pdc_server_size_input, char *checkpoint) return metadata_server_objs; } +static size_t read_checkpoint_str_len; + +#define READ_CHECKPOINT_STR(file, str_ptr) \ + do { \ + fread(&read_checkpoint_str_len, sizeof(size_t), 1, (file)); \ + printf("read_checkpoint_str_len: %d\n", read_checkpoint_str_len); \ + if (read_checkpoint_str_len > 0) { \ + (str_ptr) = PDC_calloc(1, read_checkpoint_str_len); \ + fread((str_ptr), read_checkpoint_str_len, 1, (file)); \ + } \ + } while (0) + void pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) { @@ -691,6 +710,139 @@ pdc_ls(FileNameNode *file_name_node, int argc, char *argv[]) cur_pkg = cur_pkg->next; } + // Start checkpoint region transformations + PDCtf_init_builtin_funcs(); + printf("Reading checkpoint transformations\n"); + size_t num_objs; + fread(&num_objs, sizeof(size_t), 1, file); + printf("num_objs: %lu\n", num_objs); + PDC_VECTOR *tf_obj_id_to_dg_vector_g = pdc_vector_create(PDC_MAX(num_objs, 8), 2.0); + for (int _o = 0; _o < num_objs; _o++) { + pdc_tf_obj_id_to_dg_t *cur_obj_id_to_dg = PDC_calloc(1, sizeof(pdc_tf_obj_id_to_dg_t)); + pdc_vector_add(tf_obj_id_to_dg_vector_g, cur_obj_id_to_dg); + + fread(&cur_obj_id_to_dg->obj_id, sizeof(pdcid_t), 1, file); + printf("obj_id: %d\n", cur_obj_id_to_dg->obj_id); + + char *json_filepath; + READ_CHECKPOINT_STR(file, json_filepath); + printf("\tobj[%d] json_filepath_str: %s\n", cur_obj_id_to_dg->obj_id, json_filepath); + + // Read checkpoint region mapping + size_t num_region_mappings; + fread(&num_region_mappings, sizeof(size_t), 1, file); + printf("\tnum_region_mappings: %lu\n", num_region_mappings); + cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector = + pdc_vector_create(PDC_MAX(num_region_mappings, 8), 2.0); + for (int _r = 0; _r < num_region_mappings; _r++) { + pdc_tf_region_mapping_t *cur_region_mapping = PDC_calloc(1, sizeof(pdc_tf_region_mapping_t)); + pdc_vector_add(cur_obj_id_to_dg->pdc_tf_obj.region_mappings_vector, cur_region_mapping); + + pdcid_t dg_id; + fread(&dg_id, sizeof(pdcid_t), 1, file); + cur_region_mapping->region_state.dg_id = dg_id; + printf("\t\tdg_id: %d\n", cur_region_mapping->region_state.dg_id); + + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.cur_state); + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.client_state); + READ_CHECKPOINT_STR(file, cur_region_mapping->region_state.store_state); + + printf("\t\tcur_state_str: %s\n", cur_region_mapping->region_state.cur_state); + printf("\t\tclient_state_str: %s\n", cur_region_mapping->region_state.client_state); + printf("\t\tstore_state_str: %s\n", cur_region_mapping->region_state.store_state); + + fread(&(cur_region_mapping->conceptual_region.ndim), sizeof(size_t), 1, file); + fread(&(cur_region_mapping->conceptual_region.pdc_var_type), sizeof(pdc_var_type_t), 1, file); + fread(cur_region_mapping->conceptual_region.size, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + fread(cur_region_mapping->conceptual_offset, sizeof(uint64_t), + cur_region_mapping->conceptual_region.ndim, file); + + printf("\t\tconceptual_region_ndim: %d\n", cur_region_mapping->conceptual_region.ndim); + printf("\t\tpdc_var_type: %d, size: %d\n", cur_region_mapping->conceptual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->conceptual_region.pdc_var_type)); + printf("\t\tconceptual_region_offset:\n"); + for (i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + printf("\t\t\toffset[%d]=%lu\n", i, cur_region_mapping->conceptual_offset[i]); + } + printf("\t\tconceptual_region_size:\n"); + for (i = 0; i < cur_region_mapping->conceptual_region.ndim; i++) { + printf("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->conceptual_region.size[i]); + } + + fread(&(cur_region_mapping->actual_region.ndim), sizeof(size_t), 1, file); + fread(&(cur_region_mapping->actual_region.pdc_var_type), sizeof(pdc_var_type_t), 1, file); + fread(cur_region_mapping->actual_region.size, sizeof(uint64_t), + cur_region_mapping->actual_region.ndim, file); + + printf("\t\tactual_region_ndim: %d\n", cur_region_mapping->actual_region.ndim); + printf("\t\tpdc_var_type %d, size: %d\n", cur_region_mapping->actual_region.pdc_var_type, + PDC_get_var_type_size(cur_region_mapping->actual_region.pdc_var_type)); + printf("\t\tactual_region_size:\n"); + for (i = 0; i < cur_region_mapping->actual_region.ndim; i++) { + printf("\t\t\tsize[%d]=%lu\n", i, cur_region_mapping->actual_region.size[i]); + } + } + + cur_obj_id_to_dg->dg = PDCtf_dg_json_create_common(json_filepath); + + if (cur_obj_id_to_dg == NULL) { + assert(""); + } + else if (cur_obj_id_to_dg->dg == NULL) { + assert(""); + } + + // Checkpoint state and func params for dg + for (int e_index = 0; e_index < cur_obj_id_to_dg->dg->edge_count; e_index++) { + pdc_tf_func_t *f = cur_obj_id_to_dg->dg->edges[e_index]->data; + size_t num_params; + fread(&num_params, sizeof(size_t), 1, file); + printf("\t\tfunc_name: %s\n", f->name); + printf("\t\t\tparams_str: %s\n", + (f->params_str && strlen(f->params_str) > 0) ? f->params_str : "none"); + printf("\t\t\tnum_params: %d\n", num_params); + f->pdc_tf_dg_params_vector = pdc_vector_create(PDC_MAX(num_params, 2), 2.0); + for (int _n = 0; _n < num_params; _n++) { + pdc_tf_dg_params_t *cur_param = PDC_calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(f->pdc_tf_dg_params_vector, cur_param); + + // Read conceptual_flat_offset and params_size + fread(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fread(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Read param data + cur_param->params = PDC_calloc(1, cur_param->params_size); + fread(cur_param->params, cur_param->params_size, 1, file); + + printf("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + printf("\t\t\tparams_size: %d\n", cur_param->params_size); + } + } + + for (int v_index = 0; v_index < cur_obj_id_to_dg->dg->vertex_count; v_index++) { + pdc_tf_state_t *s = cur_obj_id_to_dg->dg->vertices[v_index]->data; + size_t num_params; + fread(&num_params, sizeof(size_t), 1, file); + printf("\t\tstate_name: %s\n", s->name); + printf("\t\t\tnum_params: %d\n", num_params); + s->pdc_tf_dg_params_vector = pdc_vector_create(PDC_MAX(num_params, 2), 2.0); + for (int _n = 0; _n < num_params; _n++) { + pdc_tf_dg_params_t *cur_param = PDC_calloc(1, sizeof(pdc_tf_dg_params_t)); + pdc_vector_add(s->pdc_tf_dg_params_vector, cur_param); + + // Read conceptual_flat_offset and params_size + fread(&(cur_param->flat_conceptual_offset), sizeof(uint64_t), 1, file); + fread(&(cur_param->params_size), sizeof(uint64_t), 1, file); + // Read param data + cur_param->params = PDC_calloc(1, cur_param->params_size); + fread(cur_param->params, cur_param->params_size, 1, file); + + printf("\t\t\tconceptual_flat_offset: %lu\n", cur_param->flat_conceptual_offset); + printf("\t\t\tparams_size: %d\n", cur_param->params_size); + } + } + } + fclose(file); file = NULL; diff --git a/src/utils/include/pdc_interface.h b/src/utils/include/pdc_interface.h index e890f5ed7..2b83c5e43 100644 --- a/src/utils/include/pdc_interface.h +++ b/src/utils/include/pdc_interface.h @@ -55,8 +55,9 @@ typedef enum { PDC_CONT = 4, /* type ID for container */ PDC_OBJ = 5, /* type ID for object */ PDC_REGION = 6, /* type ID for region */ - PDC_TRANSFER_REQUEST = 7, /* type ID for region transfer */ - PDC_NTYPES = 8 /* number of library types, MUST BE LAST! */ + PDC_TRANSFER_REQUEST = 7, /* type ID for region transfer */ + PDC_TF_DG = 10, /* type ID for transformation graphs */ + PDC_NTYPES = 11 /* number of library types, MUST BE LAST! */ } PDC_type_t; /***************************/ @@ -64,12 +65,11 @@ typedef enum { /***************************/ /* ID type structure used */ struct PDC_id_type { - PDC_free_t free_func; /* Free function for object's of this type */ - PDC_type_t type_id; /* Class ID for the type */ - // const PDCID_class_t *cls;/* Pointer to ID class */ - unsigned init_count; /* # of times this type has been initialized */ - unsigned id_count; /* Current number of IDs held */ - pdcid_t nextid; /* ID to use for the next atom */ + PDC_free_t free_func; /* Free function for object's of this type */ + PDC_type_t type_id; /* Class ID for the type */ + unsigned init_count; /* Number of times this type has been initialized */ + unsigned id_count; /* Current number of IDs held */ + pdcid_t nextid; /* ID to use for the next atom */ PDC_LIST_HEAD(_pdc_id_info) ids; /* Head of list of IDs */ };