Inflight transformations

.gitignore

@@ -1,15 +1,16 @@
 # ignore swp files
 *.swp
+.cache
 
 # exes
 *.exe
-
 *.pdf
-
 src/install
 
 # ignore vscode files
 .vscode
+# ignore jetbrains
+.idea
 
 # ignore intellij files
 .idea

CMakeLists.txt

@@ -22,6 +22,143 @@ foreach(p
   endif()
 endforeach()
 
+# Dependencies directories
+# ZFP
+if(NOT DEFINED ENV{ZFP_ROOT_DIR} OR "$ENV{ZFP_ROOT_DIR}" STREQUAL "")
+    message(FATAL_ERROR "Environment variable ZFP_ROOT_DIR is not set")
+endif()
+set(ZFP_ROOT $ENV{ZFP_ROOT_DIR})
+# JSON-C
+if(NOT DEFINED ENV{JSONC_ROOT_DIR} OR "$ENV{JSONC_ROOT_DIR}" STREQUAL "")
+    message(FATAL_ERROR "Environment variable JSONC_ROOT_DIR is not set")
+endif()
+set(JSONC_ROOT $ENV{JSONC_ROOT_DIR})
+# libsodium
+if(NOT DEFINED ENV{LIBSODIUM_ROOT_DIR} OR "$ENV{LIBSODIUM_ROOT_DIR}" STREQUAL "")
+    message(FATAL_ERROR "Environment variable LIBSODIUM_ROOT_DIR is not set")
+endif()
+set(LIBSODIUM_ROOT $ENV{LIBSODIUM_ROOT_DIR})
+if(DEFINED ENV{CUDA_ROOT_DIR} AND NOT "$ENV{CUDA_ROOT_DIR}" STREQUAL "")
+  set(CUDA_ROOT $ENV{CUDA_ROOT_DIR})
+endif()
+
+# ZFP
+find_path(ZFP_INCLUDE_DIR NAMES zfp.h PATHS "${ZFP_ROOT}/include")
+find_library(ZFP_LIBRARY NAMES zfp PATHS "${ZFP_ROOT}/lib64" "${ZFP_ROOT}/lib")
+if(NOT ZFP_INCLUDE_DIR)
+  message(FATAL_ERROR "Could not find ZFP headers in ${ZFP_ROOT}/include")
+endif()
+if(NOT ZFP_LIBRARY)
+  message(FATAL_ERROR "Could not find ZFP library in ${ZFP_ROOT}/lib64 or ${ZFP_ROOT}/lib")
+endif()
+include_directories(${ZFP_INCLUDE_DIR})
+link_libraries(${ZFP_LIBRARY})
+
+# JSONC
+find_path(JSONC_INCLUDE_DIR NAMES json-c/json.h PATHS "${JSONC_ROOT}/include")
+find_library(JSONC_LIBRARY NAMES json-c PATHS "${JSONC_ROOT}/lib64" "${JSONC_ROOT}/lib")
+if(NOT JSONC_INCLUDE_DIR)
+  message(FATAL_ERROR "Could not find JSON-C headers in ${JSONC_ROOT}/include")
+endif()
+if(NOT JSONC_LIBRARY)
+  message(FATAL_ERROR "Could not find JSON-C library in ${JSONC_ROOT}/lib64 or ${LIBSODIUM_ROOT}/lib")
+endif()
+include_directories(${JSONC_INCLUDE_DIR})
+link_libraries(${JSONC_LIBRARY})
+
+# LIBSODIUM
+find_path(LIBSODIUM_INCLUDE_DIR NAMES sodium.h PATHS "${LIBSODIUM_ROOT}/include")
+find_library(LIBSODIUM_LIBRARY NAMES sodium PATHS "${LIBSODIUM_ROOT}/lib")
+if(NOT LIBSODIUM_INCLUDE_DIR)
+  message(FATAL_ERROR "Could not find libsodium headers in ${LIBSODIUM_ROOT}/include")
+endif()
+if(NOT LIBSODIUM_LIBRARY)
+  message(FATAL_ERROR "Could not find libsodium library in ${LIBSODIUM_ROOT}/lib")
+endif()
+include_directories(${LIBSODIUM_INCLUDE_DIR})
+link_libraries(${LIBSODIUM_LIBRARY})
+
+# CUDA 
+if(CUDA_ROOT)
+  # Find include directory
+  find_path(CUDA_INCLUDE_DIR NAMES cuda_runtime.h PATHS "${CUDA_ROOT}/include")
+  # Find runtime library
+  find_library(CUDA_RUNTIME_LIBRARY NAMES cudart PATHS "${CUDA_ROOT}/lib64" "${CUDA_ROOT}/lib")
+  if(NOT CUDA_INCLUDE_DIR)
+      message(FATAL_ERROR "Could not find CUDA headers in ${CUDA_ROOT}/include")
+  endif()
+  if(NOT CUDA_RUNTIME_LIBRARY)
+      message(FATAL_ERROR "Could not find CUDA runtime library in ${CUDA_ROOT}/lib64 or ${CUDA_ROOT}/lib")
+  endif()
+  include_directories(${CUDA_INCLUDE_DIR})
+  link_libraries(${CUDA_RUNTIME_LIBRARY})
+  set(CUDA_ENABLED 1 CACHE BOOL "Enable CUDA globally")
+  if(CUDA_ENABLED)
+    add_compile_definitions(CUDA_ENABLED=1)
+    add_definitions(-DUSE_CUDA)
+  endif()
+endif()
+
+if(NOT DEFINED SZ_CPU_DIR)
+  set(SZ_CPU_DIR $ENV{SZ_CPU_DIR})
+endif()
+message(STATUS "SZ_CPU_DIR = ${SZ_CPU_DIR}")
+# SZ3
+find_path(SZ3_INCLUDE_DIR NAMES sz3c.h PATHS "${SZ_CPU_DIR}/include/SZ3c")
+find_library(SZ3_LIBRARY NAMES SZ3c PATHS "${SZ_CPU_DIR}/lib64" "${SZ_CPU_DIR}/lib")
+if(NOT SZ3_INCLUDE_DIR)
+  message(FATAL_ERROR "Could not find SZ3 headers in ${SZ_CPU_DIR}/include")
+endif()
+if(NOT SZ3_LIBRARY)
+  message(FATAL_ERROR "Could not find SZ3 library in ${SZ_CPU_DIR}/lib64 or ${SZ_CPU_DIR}/lib")
+endif()
+include_directories(${SZ3_INCLUDE_DIR})
+link_libraries(${SZ3_LIBRARY})
+# Check if CUSZ_GPU_DIR is defined; if not, take from environment
+if(NOT DEFINED CUSZ_GPU_DIR)
+  set(CUSZ_GPU_DIR $ENV{CUSZ_GPU_DIR})
+endif()
+# GPU CUSZ
+if(NOT DEFINED CUSZ_GPU_DIR)
+  set(CUSZ_GPU_DIR $ENV{CUSZ_GPU_DIR})
+endif()
+message(STATUS "CUSZ_GPU_DIR = ${CUSZ_GPU_DIR}")
+# Find CUSZ headers (not SZ3)
+find_path(CUSZ_INCLUDE_DIR
+  NAMES cusz.h context.h api_v2.h header.h
+  PATHS "${CUSZ_GPU_DIR}/include/cusz"
+)
+if(NOT CUSZ_INCLUDE_DIR)
+  message(FATAL_ERROR "Could not find CUSZ headers in ${CUSZ_GPU_DIR}/include/cusz")
+endif()
+include_directories(${CUSZ_INCLUDE_DIR})
+find_library(CUSZ_LIBRARY
+  NAMES cusz
+  PATHS
+    "${CUSZ_GPU_DIR}/lib"
+    "${CUSZ_GPU_DIR}/lib64"
+)
+if(NOT CUSZ_LIBRARY)
+  message(FATAL_ERROR "Could not find CUSZ library in ${CUSZ_GPU_DIR}/lib or lib64")
+endif()
+message(STATUS "CUSZ_LIBRARY = ${CUSZ_LIBRARY}")
+link_libraries(${CUSZ_LIBRARY})
+
+# JSONC
+if(NOT DEFINED TURBO_DIR)
+  set(TURBO_DIR $ENV{TURBO_DIR})
+endif()
+find_path(TURBO_DIR_INCLUDE NAMES ic.h PATHS "${TURBO_DIR}/include")
+find_library(TURBO_DIR_LIB NAMES ic PATHS "${TURBO_DIR}")
+if(NOT TURBO_DIR_INCLUDE)
+  message(FATAL_ERROR "Could not find TURBO_DIR headers in ${TURBO_DIR}/include")
+endif()
+if(NOT TURBO_DIR_LIB)
+  message(FATAL_ERROR "Could not find TURBO_DIR library in ${TURBO_DIR}")
+endif()
+include_directories(${TURBO_DIR_INCLUDE})
+link_libraries(${TURBO_DIR_LIB})
+
 # Set a consistent MACOSX_RPATH default across all CMake versions.
 # checks if the variable CMAKE_MACOSX_RPATH is not defined and sets it to 0 if it is not defined.
 # The CMAKE_MACOSX_RPATH variable is used to control the behavior of the macOS-specific RPATH feature in CMake. 
@@ -394,10 +531,14 @@ endif()
 option(PDC_SERVER_CACHE "Enable Server Caching." ON)
 if(PDC_SERVER_CACHE)
   set(PDC_SERVER_CACHE 1)
-  set(PDC_SERVER_CACHE_MAX_GB "32" CACHE STRING "Max GB for server cache")
-  set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "3" CACHE STRING "Idle time to initiate flush from server cache")
+  set(PDC_SERVER_CACHE_MAX_GB "64" CACHE STRING "Max GB for server cache")
+  set(PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB "32" CACHE STRING "Max GB for server transformation cache")
+  set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "2" CACHE STRING "Idle time to initiate flush from server cache")
 
-  add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME})
+  add_compile_definitions(
+    PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB=${PDC_SERVER_TRANSFORMATION_CACHE_MAX_GB} 
+    PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} 
+    PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME})
 endif()
 
 
@@ -419,7 +560,16 @@ if(PDC_ENABLE_APP_CLOSE_SERVER)
 endif()
 
 #-----------------------------------------------------------------------------
-# 
+# Path to graphs for tests
+#-----------------------------------------------------------------------------
+set(TF_GRAPHS_DIR "${CMAKE_SOURCE_DIR}/tf_client/graphs/")
+if(NOT EXISTS "${TF_GRAPHS_DIR}")
+    message(FATAL_ERROR "Directory ${TF_GRAPHS_DIR} does not exist!")
+endif()
+add_compile_definitions(TF_GRAPHS_DIR="${TF_GRAPHS_DIR}")
+
+#-----------------------------------------------------------------------------
+# Wait for data fanalized in FS when object unmap is called
 #-----------------------------------------------------------------------------
 option(PDC_ENABLE_WAIT_DATA 
     "Wait for data fanalized in FS when object unmap is called." OFF)

docs/source/index.rst

@@ -1,22 +1,19 @@
 Proactive Data Containers (PDC)
 ===============================
-
 Welcome to the documentation for **Proactive Data Containers (PDC)** — 
 a high-performance data management framework designed to support flexible, scalable, 
 and efficient data movement and access across diverse memory and storage systems.
-
 Whether you're a new user, a developer integrating PDC into your application, or a 
 contributor to the project, this documentation provides the resources you need to get started and dive deeper.
-
-Here’s how to navigate:
-
+Here's how to navigate:
 - **Getting Started**  
   Learn the basics of PDC, including core concepts, architecture, and key services. Start here if you're new to PDC.
 - **Developer & API Guide**  
   Explore the client API, internal design considerations, and tools to help you develop and debug PDC-based applications.
+- **Transformation Framework**  
+  Learn how to attach compression, encryption, and custom transformation pipelines to PDC objects using directed graphs.
 - **Community and Contributions**  
   Find guidelines for contributing to PDC and additional reference material in the appendix.
-
 Use the table of contents below to explore the sections in more detail
 
 .. toctree::
@@ -34,9 +31,15 @@ Use the table of contents below to explore the sections in more detail
    client_api
    client_tools
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Data Flyway
+
+   pdc_tf_docs
+
 .. toctree::
    :maxdepth: 1
    :caption: Community and Contributions
 
    contributing
-   roadmap
+   roadmap