From 59329f23f0ad4893cc4d94f07a28563e018e74b8 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sat, 9 May 2026 20:59:33 -0400 Subject: [PATCH 01/20] Add first bits of Green function database infrastructure: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement configuration, force detection, and Butterworth-filtered STF for the Green function (GF) database (Sawade et al. 2025). This enables reciprocal simulations where stations are placed as force sources and displacement fields are recorded at earthquake element locations. Stage 1 — Configuration: - Add GF parameters to Par_file (GF_DATABASE_ENABLED, GF_DATABASE_PATH, GF_SUBSAMPLE_STEP, GF_BUFFER_SIZE, GF_NEIGHBOR_SHELLS) - Read, validate, and broadcast parameters across MPI ranks - Add GF constants to constants.h.in Stage 2 — Force Detection: - Add green_function_par module for GF runtime variables - Auto-detect force component (N/E/Z) from FORCESOLUTION direction vectors - Parse station identity (NET.STA) from FORCESOLUTION header - Validate NSOURCES=1 and USE_FORCE_POINT_SOURCE when GF enabled - Fix backward-compatible FORCESOLUTION header parsing in get_force.f90 and get_event_info.f90 (skip unused integer ID without breaking e_n) Stage 3 — STF Generation: - Implement Butterworth lowpass filter via bilinear transform (order 4, cascaded second-order sections, zero-phase forward+backward filtering) - Precompute Gaussian STF filtered at Nyquist of subsampled output - Override get_stf_viscoelastic with precomputed gf_stf(it) array - Write STF to OUTPUT_FILES/gf_stf.txt for cross-validation - Set t0 buffer to 15*hdur to accommodate filter ringing - Add unit test (test_gf_stf.f90) and cross-validation against scipy (relative error 1e-10) --- DATA/FORCESOLUTION | 11 + DATA/Par_file | 41 +- Makefile.in | 4 +- setup/constants.h.in | 11 + src/shared/broadcast_computed_parameters.f90 | 17 +- src/shared/read_parameter_file.F90 | 31 ++ src/shared/shared_par.f90 | 7 + src/specfem3D/compute_add_sources.f90 | 14 +- src/specfem3D/get_event_info.f90 | 4 +- src/specfem3D/get_force.f90 | 7 +- src/specfem3D/green_function_detect.F90 | 139 ++++++ src/specfem3D/green_function_par.F90 | 46 ++ src/specfem3D/green_function_stf.F90 | 317 +++++++++++++ src/specfem3D/prepare_timerun.F90 | 3 + src/specfem3D/rules.mk | 7 + src/specfem3D/setup_sources_receivers.f90 | 19 + tests/specfem3D/3.test_gf_stf.sh | 58 +++ tests/specfem3D/test_gf_stf.f90 | 463 +++++++++++++++++++ tests/specfem3D/test_gf_stf.makefile | 18 + 19 files changed, 1184 insertions(+), 33 deletions(-) create mode 100644 DATA/FORCESOLUTION create mode 100644 src/specfem3D/green_function_detect.F90 create mode 100644 src/specfem3D/green_function_par.F90 create mode 100644 src/specfem3D/green_function_stf.F90 create mode 100755 tests/specfem3D/3.test_gf_stf.sh create mode 100644 tests/specfem3D/test_gf_stf.f90 create mode 100644 tests/specfem3D/test_gf_stf.makefile diff --git a/DATA/FORCESOLUTION b/DATA/FORCESOLUTION new file mode 100644 index 000000000..63197fa2c --- /dev/null +++ b/DATA/FORCESOLUTION @@ -0,0 +1,11 @@ +FORCE IU.SJG +time shift: 0.0000 +f0: 0.0500 +latitude: -13.8200 +longitude: -67.2500 +depth: 647.1000 +source time function: 0 +factor force source: 1.0d15 +comp dir vect source E: 0.0 +comp dir vect source N: 1.0 +comp dir vect source Z: 0.0 diff --git a/DATA/Par_file b/DATA/Par_file index a989697dd..76344d82d 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -13,10 +13,10 @@ SAVE_FORWARD = .false. # save last frame of forward simulat NCHUNKS = 1 # angular width of the first chunk (not used if full sphere with six chunks) -ANGULAR_WIDTH_XI_IN_DEGREES = 20.d0 # angular size of a chunk -ANGULAR_WIDTH_ETA_IN_DEGREES = 20.d0 -CENTER_LATITUDE_IN_DEGREES = 40.d0 -CENTER_LONGITUDE_IN_DEGREES = 25.d0 +ANGULAR_WIDTH_XI_IN_DEGREES = 90.d0 # angular size of a chunk +ANGULAR_WIDTH_ETA_IN_DEGREES = 90.d0 +CENTER_LATITUDE_IN_DEGREES = -13.8200d0 +CENTER_LONGITUDE_IN_DEGREES = -67.2500 GAMMA_ROTATION_AZIMUTH = 0.d0 # number of elements at the surface along the two sides of the first chunk @@ -68,12 +68,12 @@ NPROC_ETA = 2 MODEL = 1D_isotropic_prem # parameters describing the Earth model -OCEANS = .true. -ELLIPTICITY = .true. -TOPOGRAPHY = .true. -GRAVITY = .true. -ROTATION = .true. -ATTENUATION = .true. +OCEANS = .false. +ELLIPTICITY = .false. +TOPOGRAPHY = .false. +GRAVITY = .false. +ROTATION = .false. +ATTENUATION = .false. # full gravity calculation by solving Poisson's equation for gravity potential instead of using a Cowling approximation # (must have also GRAVITY flag set to .true. to become active) @@ -83,7 +83,7 @@ FULL_GRAVITY = .false. POISSON_SOLVER = 0 # record length in minutes -RECORD_LENGTH_IN_MINUTES = 2.5d0 +RECORD_LENGTH_IN_MINUTES = 20.0d0 #----------------------------------------------------------- # @@ -94,11 +94,11 @@ RECORD_LENGTH_IN_MINUTES = 2.5d0 ## regional mesh cut-off # using this flag will cut-off the mesh in the mantle at a layer matching to the given cut-off depth. # this flag only has an effect for regional simulations, i.e., for NCHUNKS values less than 6. -REGIONAL_MESH_CUTOFF = .false. +REGIONAL_MESH_CUTOFF = .true. # regional mesh cut-off depth (in km) # possible selections are: 24.4d0, 80.d0, 220.d0, 400.d0, 600.d0, 670.d0, 771.d0 -REGIONAL_MESH_CUTOFF_DEPTH = 400.d0 +REGIONAL_MESH_CUTOFF_DEPTH = 771.d0 # regional mesh cut-off w/ a second doubling layer below 220km interface # (by default, a first doubling layer will be added below the Moho, and a second one below the 771km-depth layer. @@ -133,7 +133,7 @@ SPONGE_RADIUS_IN_DEGREES = 25.d0 # to undo attenuation for sensitivity kernel calculations or forward runs with SAVE_FORWARD # use one (and only one) of the two flags below. UNDO_ATTENUATION is much better (it is exact) # but requires a significant amount of disk space for temporary storage. -PARTIAL_PHYS_DISPERSION_ONLY = .true. +PARTIAL_PHYS_DISPERSION_ONLY = .false. UNDO_ATTENUATION = .false. ## undo attenuation memory @@ -298,9 +298,9 @@ SAVE_SEISMOGRAMS_STRAIN = .false. SAVE_SEISMOGRAMS_IN_ADJOINT_RUN = .false. # output format for the seismograms (one can use either or all of the three formats) -OUTPUT_SEISMOS_ASCII_TEXT = .true. +OUTPUT_SEISMOS_ASCII_TEXT = .false. OUTPUT_SEISMOS_SAC_ALPHANUM = .false. -OUTPUT_SEISMOS_SAC_BINARY = .false. +OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. OUTPUT_SEISMOS_HDF5 = .false. @@ -439,5 +439,12 @@ ADIOS_FOR_UNDO_ATTENUATION = .true. # HDF5 Database I/O # (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) -HDF5_ENABLED = .false. +HDF5_ENABLED = .true. + +# Green function database +GF_DATABASE_ENABLED = .false. +GF_DATABASE_PATH = OUTPUT_FILES/gf_database/ +GF_SUBSAMPLE_STEP = 4 +GF_BUFFER_SIZE = 100 +GF_NEIGHBOR_SHELLS = 1 diff --git a/Makefile.in b/Makefile.in index 568aa6540..baddbf583 100644 --- a/Makefile.in +++ b/Makefile.in @@ -433,7 +433,7 @@ endif @COND_ASDF_FALSE@ASDF = no #FCFLAGS += @ASDF_FCFLAGS@ -@COND_ASDF_TRUE@MPILIBS += @ASDF_LIBS@ -lasdf -lhdf5hl_fortran -lhdf5_hl -lhdf5 -lstdc++ +@COND_ASDF_TRUE@MPILIBS += @ASDF_LIBS@ -lasdf -lhdf5_hl_fortran -lhdf5_hl -lhdf5 -lstdc++ ####################################### #### @@ -516,7 +516,7 @@ endif # adds compiler flag @COND_HDF5_TRUE@FCFLAGS += @HDF5_FCFLAGS@ @HDF5_INCLUDES@ $(FC_DEFINE)USE_HDF5 -@COND_HDF5_TRUE@LDFLAGS += @HDF5_LIBS@ -lhdf5_fortran -lhdf5hl_fortran # -lhdf5_hl -lhdf5 -lstdc++ +@COND_HDF5_TRUE@LDFLAGS += @HDF5_LIBS@ -lhdf5_fortran -lhdf5_hl_fortran # -lhdf5_hl -lhdf5 -lstdc++ ####################################### ## static compilation diff --git a/setup/constants.h.in b/setup/constants.h.in index 46aadc93a..b75a3779b 100644 --- a/setup/constants.h.in +++ b/setup/constants.h.in @@ -1441,3 +1441,14 @@ ! (perturbed gravity is still computed from the density perturbation) logical, parameter :: DISCARD_GCONTRIB = .false. +!!----------------------------------------------------------- +!! +!! Green function database +!! +!!----------------------------------------------------------- + +! number of force components (N, E, Z) + integer, parameter :: GF_NCOMP_FORCE = 3 +! number of displacement dimensions (x, y, z) + integer, parameter :: GF_NCOMP_DISP = 3 + diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90 index 521083e1a..4956d309d 100644 --- a/src/shared/broadcast_computed_parameters.f90 +++ b/src/shared/broadcast_computed_parameters.f90 @@ -34,10 +34,10 @@ subroutine broadcast_computed_parameters() ! local parameters ! broadcast parameter arrays - integer, parameter :: nparam_i = 51 + integer, parameter :: nparam_i = 54 integer, dimension(nparam_i) :: bcast_integer - integer, parameter :: nparam_l = 81 + integer, parameter :: nparam_l = 82 logical, dimension(nparam_l) :: bcast_logical integer, parameter :: nparam_dp = 42 @@ -78,7 +78,8 @@ subroutine broadcast_computed_parameters() MODEL_GLL_TYPE,USER_NSTEP, & NSTEP_STEADY_STATE,NTSTEP_BETWEEN_OUTPUT_SAMPLE, & POISSON_SOLVER, & - HDF5_IO_NODES /) + HDF5_IO_NODES, & + GF_SUBSAMPLE_STEP, GF_BUFFER_SIZE, GF_NEIGHBOR_SHELLS /) bcast_logical = (/ & TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE,ANISOTROPIC_INNER_CORE, & @@ -115,7 +116,8 @@ subroutine broadcast_computed_parameters() EMC_MODEL,EMC_MODEL_TISO,EMC_MODEL_QMU, & FULL_GRAVITY, USE_SINSQ_STF, & HDF5_ENABLED, HDF5_FOR_MOVIES, OUTPUT_SEISMOS_HDF5, & - ATTENUATION_3D_BERKELEY /) + ATTENUATION_3D_BERKELEY, & + GF_DATABASE_ENABLED /) bcast_double_precision = (/ & DT, & @@ -228,6 +230,9 @@ subroutine broadcast_computed_parameters() call bcast_all_singlel(SHIFT_SIMULTANEOUS_RUNS) call bcast_all_singledp(FILESYSTEM_IO_BANDWIDTH) + ! (optional) Green function database + call bcast_all_ch(GF_DATABASE_PATH,MAX_STRING_LEN) + ! empirical minimum period resolved estimation call bcast_all_singledp(T_min_period) ! empirical minimum wavelength resolved estimation @@ -289,6 +294,9 @@ subroutine broadcast_computed_parameters() NTSTEP_BETWEEN_OUTPUT_SAMPLE = bcast_integer(49) POISSON_SOLVER = bcast_integer(50) HDF5_IO_NODES = bcast_integer(51) + GF_SUBSAMPLE_STEP = bcast_integer(52) + GF_BUFFER_SIZE = bcast_integer(53) + GF_NEIGHBOR_SHELLS = bcast_integer(54) ! logicals TRANSVERSE_ISOTROPY = bcast_logical(1) @@ -372,6 +380,7 @@ subroutine broadcast_computed_parameters() HDF5_FOR_MOVIES = bcast_logical(79) OUTPUT_SEISMOS_HDF5 = bcast_logical(80) ATTENUATION_3D_BERKELEY = bcast_logical(81) + GF_DATABASE_ENABLED = bcast_logical(82) ! double precisions DT = bcast_double_precision(1) diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index efab5452a..fb4dd5e8b 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -394,6 +394,15 @@ subroutine read_parameter_file() call read_value_integer(HDF5_IO_NODES, 'HDF5_IO_NODES', ier); ier = 0 endif + ! (optional) Green function database + call read_value_logical(GF_DATABASE_ENABLED, 'GF_DATABASE_ENABLED', ier); ier = 0 + if (GF_DATABASE_ENABLED) then + call read_value_string(GF_DATABASE_PATH, 'GF_DATABASE_PATH', ier); ier = 0 + call read_value_integer(GF_SUBSAMPLE_STEP, 'GF_SUBSAMPLE_STEP', ier); ier = 0 + call read_value_integer(GF_BUFFER_SIZE, 'GF_BUFFER_SIZE', ier); ier = 0 + call read_value_integer(GF_NEIGHBOR_SHELLS, 'GF_NEIGHBOR_SHELLS', ier); ier = 0 + endif + ! closes parameter file call close_parameter_file() @@ -440,6 +449,28 @@ subroutine read_parameter_file() print * stop 'an error occurred while reading the parameter file: HDF5 is enabled but code not built with HDF5' endif + + if (GF_DATABASE_ENABLED) then + print * + print *,'**************' + print *,'**************' + print *,'GF_DATABASE_ENABLED requires HDF5 support, but the code was not compiled with HDF5' + print *,'See --with-hdf5 configure options.' + print *,'**************' + print *,'**************' + print * + stop 'an error occurred while reading the parameter file: GF_DATABASE_ENABLED requires HDF5' + endif #endif + ! checks GF database parameter validity + if (GF_DATABASE_ENABLED) then + if (GF_SUBSAMPLE_STEP < 1) & + stop 'Error reading Par_file: GF_SUBSAMPLE_STEP must be >= 1' + if (GF_BUFFER_SIZE < 1) & + stop 'Error reading Par_file: GF_BUFFER_SIZE must be >= 1' + if (GF_NEIGHBOR_SHELLS < 0) & + stop 'Error reading Par_file: GF_NEIGHBOR_SHELLS must be >= 0' + endif + end subroutine read_parameter_file diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index cbb62f510..8d8bbb024 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -231,6 +231,13 @@ module shared_input_parameters double precision :: UCB_SOURCE_T1 = 400.d0, UCB_SOURCE_T2 = 250.d0, UCB_SOURCE_T3 = 53.d0, UCB_SOURCE_T4 = 40.d0 double precision :: UCB_TAU = 400.d0 + ! Green function database + logical :: GF_DATABASE_ENABLED = .false. + character(len=MAX_STRING_LEN) :: GF_DATABASE_PATH = 'OUTPUT_FILES/gf_database/' + integer :: GF_SUBSAMPLE_STEP = 4 + integer :: GF_BUFFER_SIZE = 100 + integer :: GF_NEIGHBOR_SHELLS = 1 + end module shared_input_parameters ! diff --git a/src/specfem3D/compute_add_sources.f90 b/src/specfem3D/compute_add_sources.f90 index af767e554..1eb14ce03 100644 --- a/src/specfem3D/compute_add_sources.f90 +++ b/src/specfem3D/compute_add_sources.f90 @@ -29,6 +29,8 @@ subroutine compute_add_sources() use specfem_par use specfem_par_crustmantle, only: accel_crust_mantle,ibool_crust_mantle + use shared_parameters, only: GF_DATABASE_ENABLED + use green_function_par, only: gf_stf implicit none @@ -76,10 +78,14 @@ subroutine compute_add_sources() timeval = time_t - tshift_src(isource) ! determines source time function value - stf = get_stf_viscoelastic(timeval,isource,it) - - ! distinguishes between single and double precision for reals - stf_used = real(stf,kind=CUSTOM_REAL) + if (GF_DATABASE_ENABLED) then + ! use precomputed Butterworth-filtered Gaussian STF + stf_used = gf_stf(it) + else + stf = get_stf_viscoelastic(timeval,isource,it) + ! distinguishes between single and double precision for reals + stf_used = real(stf,kind=CUSTOM_REAL) + endif ! adds source contribution do k = 1,NGLLZ diff --git a/src/specfem3D/get_event_info.f90 b/src/specfem3D/get_event_info.f90 index d29838cdc..9352a18a9 100644 --- a/src/specfem3D/get_event_info.f90 +++ b/src/specfem3D/get_event_info.f90 @@ -221,7 +221,9 @@ subroutine get_event_info_serial(yr,jda,mo,da,ho,mi,sec,event_name,tshift_src,t_ enddo ! read header with event information - read(string,"(a6,i4)") e_n(isource),idummy + ! format: FORCE id (e.g., FORCE 001 or FORCE IU.SJG) + ! reads first 6 chars as event name label, rest is id (discarded) + read(string,"(a6)") e_n(isource) ! read time shift read(IIN,"(a)") string diff --git a/src/specfem3D/get_force.f90 b/src/specfem3D/get_force.f90 index 6e051941b..b2267e572 100644 --- a/src/specfem3D/get_force.f90 +++ b/src/specfem3D/get_force.f90 @@ -54,9 +54,6 @@ subroutine get_force(tshift_src,hdur,lat,long,depth,DT,NSOURCES, & double precision :: length character(len=MAX_STRING_LEN) :: string character(len=MAX_STRING_LEN) :: FORCESOLUTION,path_to_add - integer :: dummyval - character(len=7) :: dummy - ! initializes lat(:) = 0.d0 long(:) = 0.d0 @@ -103,8 +100,8 @@ subroutine get_force(tshift_src,hdur,lat,long,depth,DT,NSOURCES, & ! read header with event information ! format: FORCE id - ! as example: FORCE 001 - read(string,"(a6,i4)") dummy,dummyval ! not used any further + ! as example: FORCE 001 or FORCE IU.SJG + ! (header label is not used any further, just skip) ! read time shift read(IIN,"(a)") string diff --git a/src/specfem3D/green_function_detect.F90 b/src/specfem3D/green_function_detect.F90 new file mode 100644 index 000000000..3d552b659 --- /dev/null +++ b/src/specfem3D/green_function_detect.F90 @@ -0,0 +1,139 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine gf_detect_force_component() + +! Detects which force component (N/E/Z) is being simulated and extracts +! the station identity from the FORCESOLUTION header. +! +! Called after setup_sources() when GF_DATABASE_ENABLED = .true. +! At that point, comp_dir_vect_source_E/N/Z_UP have already been read +! and broadcast to all ranks. + + use constants, only: IIN,IMAIN,MAX_STRING_LEN,TINYVAL,mygroup + + use shared_parameters, only: NSOURCES,NUMBER_OF_SIMULTANEOUS_RUNS + + use specfem_par, only: myrank, & + comp_dir_vect_source_E,comp_dir_vect_source_N,comp_dir_vect_source_Z_UP + + use green_function_par, only: gf_force_component,gf_network_name,gf_station_name + + implicit none + + ! local variables + character(len=MAX_STRING_LEN) :: string,FORCESOLUTION,path_to_add + integer :: ier,idot + logical :: has_N,has_E,has_Z + integer :: ncomp + + ! validate: GF database requires exactly one source + if (NSOURCES /= 1) then + stop 'Error: GF_DATABASE_ENABLED requires exactly 1 source (NSOURCES must be 1)' + endif + + ! detect force component from direction vectors (available on all ranks after broadcast) + has_N = abs(comp_dir_vect_source_N(1)) > TINYVAL + has_E = abs(comp_dir_vect_source_E(1)) > TINYVAL + has_Z = abs(comp_dir_vect_source_Z_UP(1)) > TINYVAL + + ncomp = 0 + if (has_N) ncomp = ncomp + 1 + if (has_E) ncomp = ncomp + 1 + if (has_Z) ncomp = ncomp + 1 + + if (ncomp /= 1) then + print *, 'Error: GF database requires exactly one nonzero force direction component' + print *, ' comp_dir_vect_source_N = ', comp_dir_vect_source_N(1) + print *, ' comp_dir_vect_source_E = ', comp_dir_vect_source_E(1) + print *, ' comp_dir_vect_source_Z_UP = ', comp_dir_vect_source_Z_UP(1) + stop 'Error: GF_DATABASE_ENABLED requires exactly one nonzero force component (N, E, or Z)' + endif + + if (has_N) gf_force_component = 1 + if (has_E) gf_force_component = 2 + if (has_Z) gf_force_component = 3 + + ! parse station identity from FORCESOLUTION header (rank 0 only, then broadcast) + if (myrank == 0) then + FORCESOLUTION = 'DATA/FORCESOLUTION' + if (NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then + write(path_to_add,"('run',i4.4,'/')") mygroup + 1 + FORCESOLUTION = path_to_add(1:len_trim(path_to_add))//FORCESOLUTION(1:len_trim(FORCESOLUTION)) + endif + + open(unit=IIN,file=trim(FORCESOLUTION),status='old',action='read',iostat=ier) + if (ier /= 0) then + print *,'Error opening file: ',trim(FORCESOLUTION) + stop 'Error opening FORCESOLUTION file for GF station detection' + endif + + ! read header line (e.g., "FORCE IU.SJG") + read(IIN,"(a)") string + ! skip empty lines + do while (len_trim(string) == 0) + read(IIN,"(a)") string + enddo + close(IIN) + + ! extract station identity: everything after "FORCE" prefix, trimmed + ! header format: "FORCE IU.SJG" or "FORCE 001" + ! skip the first 5 characters ("FORCE"), then trim leading spaces + string = adjustl(string(6:)) + + ! split on '.' to get network and station + idot = index(trim(string), '.') + if (idot > 0) then + gf_network_name = string(1:idot-1) + gf_station_name = string(idot+1:len_trim(string)) + else + ! no dot found — use entire label as station name, empty network + gf_network_name = '' + gf_station_name = trim(string) + endif + + ! user output + write(IMAIN,*) + write(IMAIN,*) 'Green function database:' + write(IMAIN,*) ' network: ', trim(gf_network_name) + write(IMAIN,*) ' station: ', trim(gf_station_name) + if (gf_force_component == 1) then + write(IMAIN,*) ' force component: N (1)' + else if (gf_force_component == 2) then + write(IMAIN,*) ' force component: E (2)' + else + write(IMAIN,*) ' force component: Z (3)' + endif + write(IMAIN,*) + call flush_IMAIN() + endif + + ! broadcast station identity to all ranks + call bcast_all_ch(gf_network_name, 8) + call bcast_all_ch(gf_station_name, 32) + + end subroutine gf_detect_force_component diff --git a/src/specfem3D/green_function_par.F90 b/src/specfem3D/green_function_par.F90 new file mode 100644 index 000000000..956a97638 --- /dev/null +++ b/src/specfem3D/green_function_par.F90 @@ -0,0 +1,46 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +module green_function_par + + use constants, only: MAX_STRING_LEN,CUSTOM_REAL + + implicit none + + ! Force component detection (Stage 2) + ! 1=N, 2=E, 3=Z + integer :: gf_force_component = 0 + character(len=8) :: gf_network_name = '' + character(len=32) :: gf_station_name = '' + + ! STF (Stage 3) + ! precomputed source time function: Gaussian filtered with Butterworth lowpass + real(kind=CUSTOM_REAL), dimension(:), allocatable :: gf_stf + double precision :: gf_f_cutoff = 0.d0 ! lowpass cutoff frequency (Hz) + double precision :: gf_hdur = 0.d0 ! half-duration used for Gaussian + +end module green_function_par diff --git a/src/specfem3D/green_function_stf.F90 b/src/specfem3D/green_function_stf.F90 new file mode 100644 index 000000000..c94840776 --- /dev/null +++ b/src/specfem3D/green_function_stf.F90 @@ -0,0 +1,317 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---------------------------------------------------------------------- +! Green function STF: Gaussian source time function filtered with +! a Butterworth lowpass at the Nyquist frequency of the subsampled output. +! +! Uses cascaded second-order sections (SOS) with forward+backward +! filtering for zero-phase response (equivalent to scipy filtfilt). +!---------------------------------------------------------------------- + + subroutine prepare_greenfunction_stf() + +! Wrapper called from prepare_timerun(). +! Checks if GF database is enabled and computes the STF. + + use shared_parameters, only: GF_DATABASE_ENABLED + + implicit none + + if (.not. GF_DATABASE_ENABLED) return + + call gf_compute_stf() + + end subroutine prepare_greenfunction_stf + +! +!---------------------------------------------------------------------- +! + + subroutine gf_compute_stf() + +! Computes a Gaussian STF filtered with a Butterworth lowpass. +! Stores the result in gf_stf(:) array (allocated here). + + use constants, only: IMAIN,PI,CUSTOM_REAL,MAX_STRING_LEN + use specfem_par, only: DT,NSTEP,t0,hdur,hdur_Gaussian,myrank,OUTPUT_FILES + use shared_parameters, only: GF_SUBSAMPLE_STEP + use green_function_par, only: gf_stf,gf_f_cutoff,gf_hdur + + implicit none + + ! local variables + integer :: it,ier + double precision :: timeval + double precision, dimension(:), allocatable :: stf_raw,stf_unfiltered + character(len=MAX_STRING_LEN) :: filename + + ! Butterworth SOS filter variables + ! For order 4: 2 second-order sections, each with 6 coefficients (b0,b1,b2,a0,a1,a2) + integer, parameter :: GF_FILTER_ORDER = 4 + integer :: nsections + double precision, dimension(GF_FILTER_ORDER/2, 6) :: sos + integer, parameter :: IOUT_STF = 71 + + ! set half-duration: use the Gaussian decay half-duration (matching specfem convention) + ! hdur_Gaussian = hdur / SOURCE_DECAY_MIMIC_TRIANGLE + ! For GF, we use hdur_Gaussian(1) since NSOURCES==1 (validated in Stage 2) + gf_hdur = hdur_Gaussian(1) + + ! compute cutoff frequency: Nyquist of subsampled output + gf_f_cutoff = 1.0d0 / (2.0d0 * DT * dble(GF_SUBSAMPLE_STEP)) + + ! allocate STF arrays + allocate(gf_stf(NSTEP)) + allocate(stf_raw(NSTEP)) + allocate(stf_unfiltered(NSTEP)) + + ! generate Gaussian STF (same formula as comp_source_time_function_gauss) + do it = 1, NSTEP + timeval = dble(it-1) * DT - t0 + stf_raw(it) = exp(-(timeval**2) / (gf_hdur**2)) / (sqrt(PI) * gf_hdur) + enddo + + ! save unfiltered for output + stf_unfiltered(:) = stf_raw(:) + + ! compute Butterworth lowpass SOS coefficients via bilinear transform + nsections = GF_FILTER_ORDER / 2 + call gf_butterworth_sos(GF_FILTER_ORDER, gf_f_cutoff, 1.0d0/DT, nsections, sos) + + ! apply zero-phase filter (forward + backward pass through each SOS) + call gf_sosfiltfilt(stf_raw, NSTEP, sos, nsections) + + ! store as CUSTOM_REAL + do it = 1, NSTEP + gf_stf(it) = real(stf_raw(it), kind=CUSTOM_REAL) + enddo + + ! write STF to OUTPUT_FILES for cross-validation + if (myrank == 0) then + filename = trim(OUTPUT_FILES) // '/gf_stf.txt' + open(unit=IOUT_STF, file=trim(filename), status='unknown', iostat=ier) + if (ier == 0) then + write(IOUT_STF, '(a)') '# Green function STF' + write(IOUT_STF, '(a,ES20.10)') '# hdur = ', gf_hdur + write(IOUT_STF, '(a,ES20.10)') '# f_cutoff = ', gf_f_cutoff + write(IOUT_STF, '(a,ES20.10)') '# dt = ', DT + write(IOUT_STF, '(a,I10)') '# nstep = ', NSTEP + write(IOUT_STF, '(a,ES20.10)') '# t0 = ', t0 + write(IOUT_STF, '(a,I10)') '# filter_order = ', GF_FILTER_ORDER + write(IOUT_STF, '(a,I10)') '# subsample_step = ', GF_SUBSAMPLE_STEP + write(IOUT_STF, '(a)') '# time(s) stf_unfiltered stf_filtered' + do it = 1, NSTEP + timeval = dble(it-1) * DT - t0 + write(IOUT_STF, '(3ES20.10)') timeval, stf_unfiltered(it), stf_raw(it) + enddo + close(IOUT_STF) + endif + endif + + deallocate(stf_raw) + deallocate(stf_unfiltered) + + ! user output + if (myrank == 0) then + write(IMAIN,*) + write(IMAIN,*) 'Green function STF:' + write(IMAIN,*) ' half-duration (Gaussian): ', sngl(gf_hdur), ' s' + write(IMAIN,*) ' lowpass cutoff frequency: ', sngl(gf_f_cutoff), ' Hz' + write(IMAIN,*) ' filter order: ', GF_FILTER_ORDER + write(IMAIN,*) ' subsample step: ', GF_SUBSAMPLE_STEP + write(IMAIN,*) ' STF peak value: ', maxval(gf_stf) + write(IMAIN,*) + endif + + end subroutine gf_compute_stf + +! +!---------------------------------------------------------------------- +! + + subroutine gf_butterworth_sos(order, fc, fs, nsections, sos) + +! Computes second-order section (SOS) coefficients for a Butterworth +! lowpass filter using the bilinear transform. +! +! Each section has 6 coefficients: [b0, b1, b2, a0, a1, a2] +! where a0 is always normalized to 1.0. + + implicit none + + double precision, parameter :: PI = 3.141592653589793238462643383279502884197d0 + + integer, intent(in) :: order, nsections + double precision, intent(in) :: fc, fs + double precision, intent(out) :: sos(nsections, 6) + + ! local variables + integer :: k + double precision :: wc, wc2 + double precision :: pole_real, pole_imag + double precision :: b0, b1, b2, a0, a1, a2 + + ! pre-warp the cutoff frequency for bilinear transform + ! wc = 2 * fs * tan(pi * fc / fs) + wc = 2.0d0 * fs * tan(PI * fc / fs) + wc2 = wc * wc + + ! for each conjugate pole pair, compute a second-order section + do k = 1, nsections + ! analog Butterworth poles (unit circle, left half plane) + ! pole angle for k-th pair: pi * (2*k + order - 1) / (2*order) + pole_real = cos(PI * dble(2*k + order - 1) / dble(2*order)) + pole_imag = sin(PI * dble(2*k + order - 1) / dble(2*order)) + + ! scale to cutoff frequency + pole_real = wc * pole_real + pole_imag = wc * pole_imag + + ! bilinear transform: s = 2*fs*(z-1)/(z+1) + ! For a conjugate pair with analog transfer function: + ! H(s) = wc^2 / (s^2 - 2*Re(p)*s + |p|^2) + ! where |p|^2 = pole_real^2 + pole_imag^2 = wc^2 (for Butterworth) + ! + ! After bilinear transform, the digital filter coefficients are: + ! b0 = wc^2 + ! b1 = 2*wc^2 + ! b2 = wc^2 + ! a0 = 4*fs^2 - 4*fs*Re(p) + |p|^2 + ! a1 = 2*|p|^2 - 8*fs^2 + ! a2 = 4*fs^2 + 4*fs*Re(p) + |p|^2 + + b0 = wc2 + b1 = 2.0d0 * wc2 + b2 = wc2 + + a0 = 4.0d0*fs*fs - 4.0d0*fs*pole_real + wc2 + a1 = 2.0d0*wc2 - 8.0d0*fs*fs + a2 = 4.0d0*fs*fs + 4.0d0*fs*pole_real + wc2 + + ! normalize so that a0 = 1 + sos(k, 1) = b0 / a0 + sos(k, 2) = b1 / a0 + sos(k, 3) = b2 / a0 + sos(k, 4) = 1.0d0 + sos(k, 5) = a1 / a0 + sos(k, 6) = a2 / a0 + enddo + + end subroutine gf_butterworth_sos + +! +!---------------------------------------------------------------------- +! + + subroutine gf_sosfiltfilt(x, n, sos, nsections) + +! Applies zero-phase filtering using cascaded second-order sections. +! For each SOS: forward pass then backward pass (like scipy filtfilt). + + implicit none + + integer, intent(in) :: n, nsections + double precision, intent(inout) :: x(n) + double precision, intent(in) :: sos(nsections, 6) + + ! local variables + integer :: isec + + do isec = 1, nsections + ! forward pass + call gf_sos_filter_forward(x, n, sos(isec, 1), sos(isec, 2), sos(isec, 3), & + sos(isec, 5), sos(isec, 6)) + ! backward pass (reverse, filter, reverse) + call gf_reverse_array(x, n) + call gf_sos_filter_forward(x, n, sos(isec, 1), sos(isec, 2), sos(isec, 3), & + sos(isec, 5), sos(isec, 6)) + call gf_reverse_array(x, n) + enddo + + end subroutine gf_sosfiltfilt + +! +!---------------------------------------------------------------------- +! + + subroutine gf_sos_filter_forward(x, n, b0, b1, b2, a1, a2) + +! Applies a single second-order section IIR filter (direct form II transposed). +! Assumes a0 = 1.0 (already normalized). +! +! Difference equation: +! y(i) = b0*x(i) + w1 +! w1 = b1*x(i) - a1*y(i) + w2 +! w2 = b2*x(i) - a2*y(i) + + implicit none + + integer, intent(in) :: n + double precision, intent(inout) :: x(n) + double precision, intent(in) :: b0, b1, b2, a1, a2 + + ! local variables + integer :: i + double precision :: w1, w2, yi + + w1 = 0.0d0 + w2 = 0.0d0 + + do i = 1, n + yi = b0 * x(i) + w1 + w1 = b1 * x(i) - a1 * yi + w2 + w2 = b2 * x(i) - a2 * yi + x(i) = yi + enddo + + end subroutine gf_sos_filter_forward + +! +!---------------------------------------------------------------------- +! + + subroutine gf_reverse_array(x, n) + +! Reverses an array in place. + + implicit none + + integer, intent(in) :: n + double precision, intent(inout) :: x(n) + + ! local variables + integer :: i + double precision :: tmp + + do i = 1, n/2 + tmp = x(i) + x(i) = x(n - i + 1) + x(n - i + 1) = tmp + enddo + + end subroutine gf_reverse_array diff --git a/src/specfem3D/prepare_timerun.F90 b/src/specfem3D/prepare_timerun.F90 index 47a736d7a..75a20d354 100644 --- a/src/specfem3D/prepare_timerun.F90 +++ b/src/specfem3D/prepare_timerun.F90 @@ -94,6 +94,9 @@ subroutine prepare_timerun() ! prepares VTK window visualization call prepare_vtk_window() + ! Green function database: precompute Butterworth-filtered STF + call prepare_greenfunction_stf() + ! optimizes array memory layout for better performance call prepare_optimized_arrays() diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index a6bf77be2..b88037480 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -86,6 +86,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/compute_strain_att.solverstatic.o \ $O/finalize_simulation.solverstatic.o \ $O/get_attenuation.solverstatic.o \ + $O/green_function_par.solverstatic_module.o \ + $O/green_function_detect.solverstatic.o \ + $O/green_function_stf.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ $O/iterate_time_undoatt.solverstatic.o \ @@ -150,6 +153,7 @@ specfem3D_SOLVER_OBJECTS += \ specfem3D_MODULES = \ $(FC_MODDIR)/asdf_data.$(FC_MODEXT) \ + $(FC_MODDIR)/green_function_par.$(FC_MODEXT) \ $(FC_MODDIR)/constants_solver.$(FC_MODEXT) \ $(FC_MODDIR)/manager_adios.$(FC_MODEXT) \ $(FC_MODDIR)/mod_element.$(FC_MODEXT) \ @@ -456,6 +460,9 @@ $O/SIEM_solver_petsc.solverstatic.o: $O/SIEM_math_library.shared.o $O/SIEM_compute_seismograms.solverstatic.o: $O/SIEM_math_library.shared.o $O/prepare_gravity.solverstatic.o: $O/SIEM_math_library.shared.o +# Green function database +$O/green_function_detect.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_stf.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index ccd98992e..17d6f37d3 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -31,6 +31,8 @@ subroutine setup_sources_receivers() TOPOGRAPHY,ibathy_topo, & USE_DISTANCE_CRITERION,xyz_midpoints,xadj,adjncy + use shared_parameters, only: GF_DATABASE_ENABLED,USE_FORCE_POINT_SOURCE + use kdtree_search, only: kdtree_delete,kdtree_nodes_location,kdtree_nodes_index implicit none @@ -41,6 +43,14 @@ subroutine setup_sources_receivers() ! locates sources and determines simulation start time t0 call setup_sources() + ! Green function database: detect force component and station identity + if (GF_DATABASE_ENABLED) then + if (.not. USE_FORCE_POINT_SOURCE) then + stop 'Error: GF_DATABASE_ENABLED requires USE_FORCE_POINT_SOURCE = .true.' + endif + call gf_detect_force_component() + endif + ! reads in stations file and locates receivers call setup_receivers() @@ -722,6 +732,7 @@ subroutine setup_stf_constants() use specfem_par use specfem_par_movie + use shared_parameters, only: GF_DATABASE_ENABLED implicit none ! local parameters @@ -825,6 +836,14 @@ subroutine setup_stf_constants() enddo endif + ! Green function database: ensure sufficient time buffer before source + ! The Butterworth lowpass filter introduces ringing that extends well + ! beyond the Gaussian's natural decay. We need a large buffer so the + ! filtered STF is essentially zero at the simulation start. + if (GF_DATABASE_ENABLED) then + t0 = max(t0, 15.0d0 * hdur(1)) + endif + ! checks if user set USER_T0 to fix simulation start time ! note: USER_T0 has to be positive if (USER_T0 > 0.d0) then diff --git a/tests/specfem3D/3.test_gf_stf.sh b/tests/specfem3D/3.test_gf_stf.sh new file mode 100755 index 000000000..8c6d20848 --- /dev/null +++ b/tests/specfem3D/3.test_gf_stf.sh @@ -0,0 +1,58 @@ +#!/bin/bash +testdir=`pwd` + +# executable +var=test_gf_stf + +# title +echo >> $testdir/results.log +echo "test: $var" >> $testdir/results.log +echo >> $testdir/results.log + +echo "directory: `pwd`" >> $testdir/results.log + +# clean +mkdir -p bin +rm -f ./bin/$var + +mkdir -p OUTPUT_FILES +rm -f OUTPUT_FILES/* + +mkdir -p DATABASES_MPI +rm -f DATABASES_MPI/* + +# single compilation +echo "compilation: $var" >> $testdir/results.log + +make -f $var.makefile $var >> $testdir/results.log 2>&1 + +echo "" >> $testdir/results.log + +# check +if [ ! -e ./bin/$var ]; then + echo "compilation of $var failed, please check..." >> $testdir/results.log + exit 1 +fi + +# runs test +echo "run: `date`" >> $testdir/results.log +./bin/$var >> $testdir/results.log 2>$testdir/error.log + +# checks exit code +if [[ $? -ne 0 ]]; then + echo "test failed"; echo "error log:"; cat $testdir/error.log; echo "" + exit 1 +fi + +# checks error output (note: fortran stop returns with a zero-exit code) +if [[ -s $testdir/error.log ]]; then + echo "returned ERROR output:" >> $testdir/results.log + cat $testdir/error.log >> $testdir/results.log + exit 1 +fi +rm -f $testdir/error.log + +#cleanup +rm -f bin/$var +# done +echo "successfully tested: `date`" >> $testdir/results.log diff --git a/tests/specfem3D/test_gf_stf.f90 b/tests/specfem3D/test_gf_stf.f90 new file mode 100644 index 000000000..5f449a132 --- /dev/null +++ b/tests/specfem3D/test_gf_stf.f90 @@ -0,0 +1,463 @@ +program test_gf_stf + +! Unit test for Green function STF: Butterworth filter + Gaussian STF. +! +! Tests: +! 1. Butterworth SOS coefficient computation +! 2. Filter application (lowpass removes high frequencies) +! 3. Full STF generation: Gaussian + Butterworth lowpass +! 4. STF properties: peak location, area, frequency content +! 5. Write output for Python cross-validation + + implicit none + + double precision, parameter :: PI = 3.141592653589793d0 + + ! test parameters + double precision, parameter :: dt = 0.05d0 ! sampling interval (s) + integer, parameter :: nstep = 2000 ! number of timesteps + double precision, parameter :: hdur = 0.25d0 ! Gaussian half-duration (s) + double precision, parameter :: f_cutoff = 2.5d0 ! lowpass cutoff (Hz) + double precision, parameter :: fs = 1.0d0 / dt ! sampling rate (Hz) + integer, parameter :: filter_order = 4 + integer, parameter :: nsections = filter_order / 2 + + ! arrays + double precision :: sos(nsections, 6) + double precision :: stf(nstep), stf_filtered(nstep) + double precision :: t(nstep) + + ! test variables + double precision :: peak_val, peak_time, area + double precision :: timeval + integer :: i, ipeak + integer :: nfail + + ! for frequency content check + integer :: nfft, k + double precision :: freq, amp_sum_above, amp_sum_below + double precision, allocatable :: fft_real(:), fft_imag(:) + + ! output file + integer, parameter :: IOUT = 20 + + nfail = 0 + + print *, 'program: test_gf_stf' + print * + + !-------------------------------------------------------------------- + ! Test 1: Butterworth SOS coefficients + !-------------------------------------------------------------------- + print *, 'Test 1: Butterworth SOS coefficients' + + call gf_butterworth_sos(filter_order, f_cutoff, fs, nsections, sos) + + ! verify basic properties + do i = 1, nsections + print *, ' Section ', i + print *, ' b = ', sos(i,1), sos(i,2), sos(i,3) + print *, ' a = ', sos(i,4), sos(i,5), sos(i,6) + + ! a0 must be 1.0 + if (abs(sos(i,4) - 1.0d0) > 1.0d-12) then + print *, ' FAIL: a0 is not 1.0' + nfail = nfail + 1 + endif + + ! b0, b2 must be positive (lowpass) + if (sos(i,1) <= 0.0d0 .or. sos(i,3) <= 0.0d0) then + print *, ' FAIL: b0 or b2 is not positive' + nfail = nfail + 1 + endif + + ! b1 = 2*b0 for lowpass Butterworth + if (abs(sos(i,2) - 2.0d0*sos(i,1)) > 1.0d-12) then + print *, ' FAIL: b1 != 2*b0' + nfail = nfail + 1 + endif + enddo + + ! DC gain check: product of section DC gains should be 1.0 + call check_dc_gain(sos, nsections, nfail) + + print *, ' done.' + print * + + !-------------------------------------------------------------------- + ! Test 2: Lowpass filter removes high frequencies + !-------------------------------------------------------------------- + print *, 'Test 2: Lowpass filter effect on impulse' + + ! create an impulse signal + stf(:) = 0.0d0 + stf(nstep/2) = 1.0d0 + + ! apply filter + stf_filtered(:) = stf(:) + call gf_sosfiltfilt(stf_filtered, nstep, sos, nsections) + + ! the filtered impulse should be spread out and have lower peak + if (maxval(abs(stf_filtered)) >= 1.0d0) then + print *, ' FAIL: filtered impulse peak should be less than 1.0, got ', maxval(abs(stf_filtered)) + nfail = nfail + 1 + else + print *, ' filtered impulse peak: ', maxval(abs(stf_filtered)), ' (< 1.0, OK)' + endif + + ! the filtered signal should be symmetric (zero-phase) + call check_symmetry(stf_filtered, nstep, nstep/2, nfail) + + print *, ' done.' + print * + + !-------------------------------------------------------------------- + ! Test 3: Full Gaussian + Butterworth STF + !-------------------------------------------------------------------- + print *, 'Test 3: Gaussian + Butterworth STF' + + ! generate time array and Gaussian STF + ! center the Gaussian at t = 5*hdur + do i = 1, nstep + t(i) = dble(i-1) * dt + timeval = t(i) - 5.0d0 * hdur + stf(i) = exp(-(timeval**2) / (hdur**2)) / (sqrt(PI) * hdur) + enddo + + ! make a copy and filter it + stf_filtered(:) = stf(:) + call gf_sosfiltfilt(stf_filtered, nstep, sos, nsections) + + ! check peak location (should be near 5*hdur) + ipeak = 1 + peak_val = stf_filtered(1) + do i = 2, nstep + if (stf_filtered(i) > peak_val) then + peak_val = stf_filtered(i) + ipeak = i + endif + enddo + peak_time = t(ipeak) + + print *, ' peak value: ', peak_val + print *, ' peak time: ', peak_time, ' s (expected ~', 5.0d0*hdur, ' s)' + + if (abs(peak_time - 5.0d0*hdur) > 2.0d0*dt) then + print *, ' FAIL: peak time shifted too much' + nfail = nfail + 1 + endif + + ! check area (trapezoidal integration, should be ~1.0 for normalized Gaussian) + area = 0.0d0 + do i = 1, nstep - 1 + area = area + 0.5d0 * (stf_filtered(i) + stf_filtered(i+1)) * dt + enddo + print *, ' area (integral): ', area, ' (expected ~1.0)' + + if (abs(area - 1.0d0) > 0.05d0) then + print *, ' FAIL: area deviates from 1.0 by more than 5%' + nfail = nfail + 1 + endif + + ! check peak is positive + if (peak_val <= 0.0d0) then + print *, ' FAIL: peak value should be positive' + nfail = nfail + 1 + endif + + print *, ' done.' + print * + + !-------------------------------------------------------------------- + ! Test 4: Frequency content check (DFT) + !-------------------------------------------------------------------- + print *, 'Test 4: Frequency content of filtered STF' + + nfft = nstep + allocate(fft_real(nfft), fft_imag(nfft)) + + ! compute DFT of filtered STF + call compute_dft(stf_filtered, nstep, fft_real, fft_imag, nfft) + + ! compute total energy below and above cutoff + amp_sum_below = 0.0d0 + amp_sum_above = 0.0d0 + do k = 1, nfft/2 + freq = dble(k-1) * fs / dble(nfft) + if (freq > 0.0d0 .and. freq <= f_cutoff) then + amp_sum_below = amp_sum_below + fft_real(k)**2 + fft_imag(k)**2 + else if (freq > f_cutoff) then + amp_sum_above = amp_sum_above + fft_real(k)**2 + fft_imag(k)**2 + endif + enddo + + print *, ' energy below cutoff: ', amp_sum_below + print *, ' energy above cutoff: ', amp_sum_above + + ! energy above cutoff should be negligible compared to below + if (amp_sum_below > 0.0d0) then + if (amp_sum_above / amp_sum_below > 1.0d-4) then + print *, ' FAIL: too much energy above cutoff (ratio = ', amp_sum_above/amp_sum_below, ')' + nfail = nfail + 1 + else + print *, ' energy ratio above/below cutoff: ', amp_sum_above/amp_sum_below, ' (< 1e-4, OK)' + endif + endif + + deallocate(fft_real, fft_imag) + + print *, ' done.' + print * + + !-------------------------------------------------------------------- + ! Test 5: Write output for Python cross-validation + !-------------------------------------------------------------------- + print *, 'Writing STF to test_stf_output.txt' + + ! regenerate unfiltered for comparison + do i = 1, nstep + timeval = t(i) - 5.0d0 * hdur + stf(i) = exp(-(timeval**2) / (hdur**2)) / (sqrt(PI) * hdur) + enddo + + open(unit=IOUT, file='test_stf_output.txt', status='unknown') + write(IOUT, '(a)') '# Green function STF unit test output' + write(IOUT, '(a,ES20.10)') '# hdur = ', hdur + write(IOUT, '(a,ES20.10)') '# f_cutoff = ', f_cutoff + write(IOUT, '(a,ES20.10)') '# dt = ', dt + write(IOUT, '(a,I10)') '# nstep = ', nstep + write(IOUT, '(a,ES20.10)') '# t_center = ', 5.0d0*hdur + write(IOUT, '(a,I10)') '# filter_order = ', filter_order + write(IOUT, '(a)') '# time(s) stf_unfiltered stf_filtered' + do i = 1, nstep + write(IOUT, '(3ES20.10)') t(i), stf(i), stf_filtered(i) + enddo + close(IOUT) + + print *, ' done.' + print * + + !-------------------------------------------------------------------- + ! Summary + !-------------------------------------------------------------------- + if (nfail == 0) then + print *, '--- all tests passed ---' + else + print *, '--- FAILED: ', nfail, ' test(s) ---' + stop 1 + endif + +end program test_gf_stf + +!====================================================================== +! Helper subroutines for the test +!====================================================================== + +subroutine check_dc_gain(sos, nsections, nfail) + + implicit none + integer, intent(in) :: nsections + double precision, intent(in) :: sos(nsections, 6) + integer, intent(inout) :: nfail + + double precision :: dc_gain, section_gain + integer :: i + + dc_gain = 1.0d0 + do i = 1, nsections + section_gain = (sos(i,1) + sos(i,2) + sos(i,3)) / & + (sos(i,4) + sos(i,5) + sos(i,6)) + dc_gain = dc_gain * section_gain + enddo + + print *, ' total DC gain: ', dc_gain, ' (expected 1.0)' + + if (abs(dc_gain - 1.0d0) > 1.0d-10) then + print *, ' FAIL: DC gain is not 1.0' + nfail = nfail + 1 + endif + +end subroutine check_dc_gain + +!---------------------------------------------------------------------- + +subroutine check_symmetry(x, n, icenter, nfail) + + implicit none + integer, intent(in) :: n, icenter + double precision, intent(in) :: x(n) + integer, intent(inout) :: nfail + + integer :: i, ncheck + double precision :: max_asym + + ! check symmetry around icenter + ncheck = min(icenter - 1, n - icenter) + ncheck = min(ncheck, 100) + + max_asym = 0.0d0 + do i = 1, ncheck + max_asym = max(max_asym, abs(x(icenter + i) - x(icenter - i))) + enddo + + print *, ' max asymmetry around center: ', max_asym + + if (max_asym > 1.0d-12) then + print *, ' FAIL: filtered impulse is not symmetric (zero-phase violated)' + nfail = nfail + 1 + endif + +end subroutine check_symmetry + +!---------------------------------------------------------------------- + +subroutine compute_dft(x, n, dft_real, dft_imag, nfft) + +! Simple DFT computation (not FFT, but sufficient for testing) + + implicit none + + double precision, parameter :: PI = 3.141592653589793d0 + + integer, intent(in) :: n, nfft + double precision, intent(in) :: x(n) + double precision, intent(out) :: dft_real(nfft), dft_imag(nfft) + + integer :: k, j + double precision :: angle + + do k = 1, nfft + dft_real(k) = 0.0d0 + dft_imag(k) = 0.0d0 + do j = 1, n + angle = 2.0d0 * PI * dble(k-1) * dble(j-1) / dble(nfft) + dft_real(k) = dft_real(k) + x(j) * cos(angle) + dft_imag(k) = dft_imag(k) - x(j) * sin(angle) + enddo + enddo + +end subroutine compute_dft + +!====================================================================== +! Filter subroutines (identical to src/specfem3D/green_function_stf.F90) +!====================================================================== + +subroutine gf_butterworth_sos(order, fc, fs, nsections, sos) + +! Computes second-order section (SOS) coefficients for a Butterworth +! lowpass filter using the bilinear transform. + + implicit none + + double precision, parameter :: PI = 3.141592653589793238462643383279502884197d0 + + integer, intent(in) :: order, nsections + double precision, intent(in) :: fc, fs + double precision, intent(out) :: sos(nsections, 6) + + integer :: k + double precision :: wc, wc2 + double precision :: pole_real, pole_imag + double precision :: b0, b1, b2, a0, a1, a2 + + wc = 2.0d0 * fs * tan(PI * fc / fs) + wc2 = wc * wc + + do k = 1, nsections + pole_real = cos(PI * dble(2*k + order - 1) / dble(2*order)) + pole_imag = sin(PI * dble(2*k + order - 1) / dble(2*order)) + + pole_real = wc * pole_real + pole_imag = wc * pole_imag + + b0 = wc2 + b1 = 2.0d0 * wc2 + b2 = wc2 + + a0 = 4.0d0*fs*fs - 4.0d0*fs*pole_real + wc2 + a1 = 2.0d0*wc2 - 8.0d0*fs*fs + a2 = 4.0d0*fs*fs + 4.0d0*fs*pole_real + wc2 + + sos(k, 1) = b0 / a0 + sos(k, 2) = b1 / a0 + sos(k, 3) = b2 / a0 + sos(k, 4) = 1.0d0 + sos(k, 5) = a1 / a0 + sos(k, 6) = a2 / a0 + enddo + +end subroutine gf_butterworth_sos + +!---------------------------------------------------------------------- + +subroutine gf_sosfiltfilt(x, n, sos, nsections) + +! Applies zero-phase filtering using cascaded second-order sections. + + implicit none + + integer, intent(in) :: n, nsections + double precision, intent(inout) :: x(n) + double precision, intent(in) :: sos(nsections, 6) + + integer :: isec + + do isec = 1, nsections + call gf_sos_filter_forward(x, n, sos(isec, 1), sos(isec, 2), sos(isec, 3), & + sos(isec, 5), sos(isec, 6)) + call gf_reverse_array(x, n) + call gf_sos_filter_forward(x, n, sos(isec, 1), sos(isec, 2), sos(isec, 3), & + sos(isec, 5), sos(isec, 6)) + call gf_reverse_array(x, n) + enddo + +end subroutine gf_sosfiltfilt + +!---------------------------------------------------------------------- + +subroutine gf_sos_filter_forward(x, n, b0, b1, b2, a1, a2) + +! Single second-order section IIR filter (direct form II transposed). + + implicit none + + integer, intent(in) :: n + double precision, intent(inout) :: x(n) + double precision, intent(in) :: b0, b1, b2, a1, a2 + + integer :: i + double precision :: w1, w2, yi + + w1 = 0.0d0 + w2 = 0.0d0 + + do i = 1, n + yi = b0 * x(i) + w1 + w1 = b1 * x(i) - a1 * yi + w2 + w2 = b2 * x(i) - a2 * yi + x(i) = yi + enddo + +end subroutine gf_sos_filter_forward + +!---------------------------------------------------------------------- + +subroutine gf_reverse_array(x, n) + +! Reverses an array in place. + + implicit none + + integer, intent(in) :: n + double precision, intent(inout) :: x(n) + + integer :: i + double precision :: tmp + + do i = 1, n/2 + tmp = x(i) + x(i) = x(n - i + 1) + x(n - i + 1) = tmp + enddo + +end subroutine gf_reverse_array diff --git a/tests/specfem3D/test_gf_stf.makefile b/tests/specfem3D/test_gf_stf.makefile new file mode 100644 index 000000000..3d0a7b34b --- /dev/null +++ b/tests/specfem3D/test_gf_stf.makefile @@ -0,0 +1,18 @@ +# includes default Makefile from previous configuration +include Makefile + +# test target +default: test_gf_stf + +## compilation directories +O := ./obj + +OBJECTS = \ + $(EMPTY_MACRO) + +# The test links against the filter subroutines extracted into a local copy. +# The main green_function_stf.F90 has specfem module dependencies that we +# don't need for pure filter testing. +test_gf_stf: + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -o ./bin/test_gf_stf test_gf_stf.f90 $(LDFLAGS) $(LIBS) + From da5d562799e8031cb9aefe487ddde88d91f75671 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sat, 9 May 2026 21:31:46 -0400 Subject: [PATCH 02/20] Add GF element location (Stage 4): read DATA/GF_LOCATIONS and locate elements - New green_function_locate.F90: gf_read_locations() parses lat/lon/depth_km from DATA/GF_LOCATIONS; gf_locate_elements() converts geographic coords to Cartesian (with ellipticity/topography), uses KD-tree + locate_MPI_slice to find containing elements across MPI ranks, then deduplicates with O(n) boolean mask per rank. - Errors if GF_LOCATIONS is missing or empty when GF_DATABASE_ENABLED=.true. - Hooked into setup_sources_receivers.f90 before xadj/adjncy deallocation (required for Stage 5 neighbor expansion). - Updated green_function_par.F90 with location arrays and rules.mk with build target. --- src/specfem3D/green_function_locate.F90 | 407 ++++++++++++++++++++++ src/specfem3D/green_function_par.F90 | 8 + src/specfem3D/rules.mk | 2 + src/specfem3D/setup_sources_receivers.f90 | 7 + 4 files changed, 424 insertions(+) create mode 100644 src/specfem3D/green_function_locate.F90 diff --git a/src/specfem3D/green_function_locate.F90 b/src/specfem3D/green_function_locate.F90 new file mode 100644 index 000000000..388248450 --- /dev/null +++ b/src/specfem3D/green_function_locate.F90 @@ -0,0 +1,407 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---- +!---- Green function database: read GF_LOCATIONS and locate containing elements +!---- + + subroutine gf_read_locations() + +! Reads DATA/GF_LOCATIONS file on rank 0 and broadcasts to all ranks. +! Each line: latitude longitude depth_km +! Lines starting with '#' are comments, empty lines are skipped. + + use constants, only: IMAIN,MAX_STRING_LEN,IIN + use specfem_par, only: myrank + + use green_function_par, only: gf_nlocations,gf_lat,gf_lon,gf_depth + + implicit none + + ! local parameters + integer :: ier,iline,igf + character(len=MAX_STRING_LEN) :: line + character(len=MAX_STRING_LEN) :: gf_locations_file + double precision :: lat,lon,depth_km + + ! counts valid lines and reads coordinates on main process + gf_locations_file = 'DATA/GF_LOCATIONS' + + if (myrank == 0) then + ! first pass: count valid lines + gf_nlocations = 0 + open(unit=IIN,file=trim(gf_locations_file),status='old',action='read',iostat=ier) + if (ier /= 0) then + print *, 'Error: GF_DATABASE_ENABLED is .true. but could not open ',trim(gf_locations_file) + call exit_MPI(myrank,'GF_DATABASE_ENABLED requires a DATA/GF_LOCATIONS file') + else + do + read(IIN,'(a)',iostat=ier) line + if (ier /= 0) exit + ! skip comments and blank lines + line = adjustl(line) + if (len_trim(line) == 0) cycle + if (line(1:1) == '#') cycle + gf_nlocations = gf_nlocations + 1 + enddo + close(IIN) + endif + + write(IMAIN,*) + write(IMAIN,*) '********************' + write(IMAIN,*) ' Green function database: reading GF_LOCATIONS' + write(IMAIN,*) '********************' + write(IMAIN,*) + write(IMAIN,*) 'Number of GF locations: ',gf_nlocations + call flush_IMAIN() + endif + + ! broadcast count + call bcast_all_singlei(gf_nlocations) + + ! error if no locations found + if (gf_nlocations == 0) then + if (myrank == 0) print *, 'Error: GF_LOCATIONS file has no valid entries' + call exit_MPI(myrank,'GF_DATABASE_ENABLED requires at least one location in DATA/GF_LOCATIONS') + endif + + ! allocate coordinate arrays on all ranks + allocate(gf_lat(gf_nlocations), & + gf_lon(gf_nlocations), & + gf_depth(gf_nlocations),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating GF location arrays') + + ! second pass: read coordinates on main + if (myrank == 0) then + open(unit=IIN,file=trim(gf_locations_file),status='old',action='read',iostat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error re-opening GF_LOCATIONS file') + + igf = 0 + do + read(IIN,'(a)',iostat=ier) line + if (ier /= 0) exit + line = adjustl(line) + if (len_trim(line) == 0) cycle + if (line(1:1) == '#') cycle + + igf = igf + 1 + read(line,*,iostat=ier) lat, lon, depth_km + if (ier /= 0) then + write(IMAIN,*) 'Error parsing GF_LOCATIONS line ',igf,': ',trim(line) + call exit_MPI(myrank,'Error reading GF_LOCATIONS coordinates') + endif + + gf_lat(igf) = lat + gf_lon(igf) = lon + gf_depth(igf) = depth_km + enddo + close(IIN) + + ! print locations + do igf = 1,gf_nlocations + write(IMAIN,'(a,i6,a,f10.4,a,f10.4,a,f10.2,a)') & + ' Location #',igf, & + ': lat=',gf_lat(igf), & + ' lon=',gf_lon(igf), & + ' depth=',gf_depth(igf),' km' + enddo + write(IMAIN,*) + call flush_IMAIN() + endif + + ! broadcast coordinates to all ranks + call bcast_all_dp(gf_lat,gf_nlocations) + call bcast_all_dp(gf_lon,gf_nlocations) + call bcast_all_dp(gf_depth,gf_nlocations) + + end subroutine gf_read_locations + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_locate_elements() + +! Locates which spectral elements contain the GF locations. +! Follows the pattern of locate_receivers.f90 but only needs ispec/islice, +! not interpolation coefficients. +! After locating, builds a unique list of tagged elements per rank using +! O(n) boolean mask deduplication. + + use constants_solver, only: & + ELLIPTICITY_VAL,NDIM,HUGEVAL,IMAIN, & + DEGREES_TO_RADIANS,R_UNIT_SPHERE, & + nrec_SUBSET_MAX + + use shared_parameters, only: R_PLANET + + use specfem_par, only: & + myrank, & + nspec => NSPEC_CRUST_MANTLE, & + ibathy_topo,TOPOGRAPHY + + use specfem_par, only: rspl_ellip,ellipicity_spline,ellipicity_spline2,nspl_ellip + + use green_function_par, only: & + gf_nlocations,gf_lat,gf_lon,gf_depth, & + gf_ispec_selected,gf_islice_selected, & + gf_nelem_local,gf_local_elements + + implicit none + + ! local parameters + integer :: igf,igf_in_this_subset,igf_already_done,ier + integer :: ngf_SUBSET_current_size + double precision :: lat,lon,depth_km,depth_m + double precision :: theta,phi + double precision :: sint,cost,sinp,cosp + double precision :: r0,r_target,elevation + double precision :: x_target,y_target,z_target + double precision :: x,y,z + double precision :: xi,eta,gamma + double precision :: distmin_not_squared + integer :: ispec_selected,ispec,j + + ! subset arrays for locate_MPI_slice + integer, allocatable, dimension(:) :: ispec_selected_subset + double precision, allocatable, dimension(:,:) :: xyz_found_subset + double precision, allocatable, dimension(:) :: xi_subset,eta_subset,gamma_subset + double precision, allocatable, dimension(:) :: final_distance_subset + + ! full arrays for MPI gathering + double precision, allocatable, dimension(:,:) :: xyz_found + double precision, allocatable, dimension(:) :: xi_all,eta_all,gamma_all + double precision, allocatable, dimension(:) :: final_distance + + ! deduplication mask + logical, allocatable, dimension(:) :: ispec_tagged + + logical, parameter :: POINT_CAN_BE_BURIED = .true. + + ! nothing to do + if (gf_nlocations == 0) then + gf_nelem_local = 0 + return + endif + + if (myrank == 0) then + write(IMAIN,*) + write(IMAIN,*) '********************' + write(IMAIN,*) ' Green function database: locating elements' + write(IMAIN,*) '********************' + write(IMAIN,*) + call flush_IMAIN() + endif + + ! allocate result arrays + allocate(gf_ispec_selected(gf_nlocations), & + gf_islice_selected(gf_nlocations), & + xyz_found(NDIM,gf_nlocations), & + xi_all(gf_nlocations), & + eta_all(gf_nlocations), & + gamma_all(gf_nlocations), & + final_distance(gf_nlocations),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating GF element location arrays') + + gf_ispec_selected(:) = 0 + gf_islice_selected(:) = -1 + xyz_found(:,:) = 0.d0 + xi_all(:) = 0.d0 + eta_all(:) = 0.d0 + gamma_all(:) = 0.d0 + final_distance(:) = HUGEVAL + + ! loop over subsets (same pattern as locate_receivers) + do igf_already_done = 0, gf_nlocations, nrec_SUBSET_MAX + + ngf_SUBSET_current_size = min(nrec_SUBSET_MAX, gf_nlocations - igf_already_done) + if (ngf_SUBSET_current_size <= 0) exit + + ! allocate subset arrays + allocate(ispec_selected_subset(ngf_SUBSET_current_size), & + xi_subset(ngf_SUBSET_current_size), & + eta_subset(ngf_SUBSET_current_size), & + gamma_subset(ngf_SUBSET_current_size), & + xyz_found_subset(NDIM,ngf_SUBSET_current_size), & + final_distance_subset(ngf_SUBSET_current_size),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating GF subset arrays') + + ispec_selected_subset(:) = 0 + final_distance_subset(:) = HUGEVAL + + ! locate points in this subset + do igf_in_this_subset = 1, ngf_SUBSET_current_size + igf = igf_in_this_subset + igf_already_done + + lat = gf_lat(igf) + lon = gf_lon(igf) + depth_km = gf_depth(igf) + + ! depth in meters + depth_m = depth_km * 1000.d0 + + ! normalize longitude to [0, 360] + if (lon < 0.d0) lon = lon + 360.d0 + if (lon > 360.d0) lon = lon - 360.d0 + + ! geographic latitude -> geocentric colatitude (radians) + call lat_2_geocentric_colat_dble(lat,theta,ELLIPTICITY_VAL) + + ! longitude to radians + phi = lon * DEGREES_TO_RADIANS + + ! reduce to valid ranges + call reduce(theta,phi) + + sint = sin(theta) + cost = cos(theta) + sinp = sin(phi) + cosp = cos(phi) + + ! start with unit sphere radius + r0 = R_UNIT_SPHERE + + ! topography + if (TOPOGRAPHY) then + call get_topo_bathy(lat,lon,elevation,ibathy_topo) + r0 = r0 + elevation / R_PLANET + endif + + ! ellipticity + if (ELLIPTICITY_VAL) then + call add_ellipticity_rtheta(r0,theta,nspl_ellip,rspl_ellip, & + ellipicity_spline,ellipicity_spline2) + endif + + ! subtract depth (in meters) + r0 = r0 - depth_m / R_PLANET + r_target = r0 + + ! Cartesian coordinates + x_target = r_target * sint * cosp + y_target = r_target * sint * sinp + z_target = r_target * cost + + ! locate element + call locate_point(x_target,y_target,z_target,lat,lon, & + ispec_selected,xi,eta,gamma, & + x,y,z,distmin_not_squared,POINT_CAN_BE_BURIED) + + ! store results + ispec_selected_subset(igf_in_this_subset) = ispec_selected + xi_subset(igf_in_this_subset) = xi + eta_subset(igf_in_this_subset) = eta + gamma_subset(igf_in_this_subset) = gamma + xyz_found_subset(1,igf_in_this_subset) = x + xyz_found_subset(2,igf_in_this_subset) = y + xyz_found_subset(3,igf_in_this_subset) = z + final_distance_subset(igf_in_this_subset) = distmin_not_squared + + enddo + + ! gather across MPI ranks to find best location + call locate_MPI_slice(ngf_SUBSET_current_size,igf_already_done, & + ispec_selected_subset, & + xyz_found_subset, & + xi_subset,eta_subset,gamma_subset, & + final_distance_subset, & + gf_nlocations,gf_ispec_selected,gf_islice_selected, & + xyz_found, & + xi_all,eta_all,gamma_all, & + final_distance) + + deallocate(ispec_selected_subset) + deallocate(xi_subset,eta_subset,gamma_subset) + deallocate(final_distance_subset) + deallocate(xyz_found_subset) + + enddo + + ! user output on main + if (myrank == 0) then + do igf = 1,gf_nlocations + write(IMAIN,'(a,i6,a,f10.4,a,f10.4,a,f10.2,a,i6,a,i8,a,f8.3,a)') & + ' GF location #',igf, & + ': lat=',gf_lat(igf), & + ' lon=',gf_lon(igf), & + ' depth=',gf_depth(igf), & + ' km -> slice ',gf_islice_selected(igf), & + ' element ',gf_ispec_selected(igf), & + ' (dist=',sngl(final_distance(igf)),' km)' + enddo + write(IMAIN,*) + call flush_IMAIN() + endif + + ! broadcast results to all ranks + call bcast_all_i(gf_ispec_selected,gf_nlocations) + call bcast_all_i(gf_islice_selected,gf_nlocations) + + ! free temporary arrays + deallocate(xyz_found) + deallocate(xi_all,eta_all,gamma_all) + deallocate(final_distance) + + ! O(n) deduplication: build unique element list per rank using boolean mask + allocate(ispec_tagged(nspec),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating ispec_tagged') + ispec_tagged(:) = .false. + + do igf = 1, gf_nlocations + if (gf_islice_selected(igf) == myrank) then + ispec_tagged(gf_ispec_selected(igf)) = .true. + endif + enddo + + ! count unique elements + gf_nelem_local = count(ispec_tagged) + + ! pack into compact array + if (gf_nelem_local > 0) then + allocate(gf_local_elements(gf_nelem_local),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating gf_local_elements') + + j = 0 + do ispec = 1, nspec + if (ispec_tagged(ispec)) then + j = j + 1 + gf_local_elements(j) = ispec + endif + enddo + endif + + deallocate(ispec_tagged) + + ! report per-rank element counts + if (myrank == 0) then + write(IMAIN,*) 'Green function database: element tagging summary' + write(IMAIN,*) ' Rank 0 has ',gf_nelem_local,' unique tagged elements' + write(IMAIN,*) + call flush_IMAIN() + endif + + end subroutine gf_locate_elements diff --git a/src/specfem3D/green_function_par.F90 b/src/specfem3D/green_function_par.F90 index 956a97638..a7622286f 100644 --- a/src/specfem3D/green_function_par.F90 +++ b/src/specfem3D/green_function_par.F90 @@ -43,4 +43,12 @@ module green_function_par double precision :: gf_f_cutoff = 0.d0 ! lowpass cutoff frequency (Hz) double precision :: gf_hdur = 0.d0 ! half-duration used for Gaussian + ! Element location (Stage 4) + integer :: gf_nlocations = 0 ! total GF locations read + double precision, dimension(:), allocatable :: gf_lat, gf_lon, gf_depth ! location coords + integer, dimension(:), allocatable :: gf_ispec_selected ! element index per location + integer, dimension(:), allocatable :: gf_islice_selected ! MPI rank per location + integer :: gf_nelem_local = 0 ! unique tagged elements on this rank + integer, dimension(:), allocatable :: gf_local_elements ! unique local element indices + end module green_function_par diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index b88037480..764e99e49 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -89,6 +89,7 @@ specfem3D_SOLVER_OBJECTS += \ $O/green_function_par.solverstatic_module.o \ $O/green_function_detect.solverstatic.o \ $O/green_function_stf.solverstatic.o \ + $O/green_function_locate.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ $O/iterate_time_undoatt.solverstatic.o \ @@ -463,6 +464,7 @@ $O/prepare_gravity.solverstatic.o: $O/SIEM_math_library.shared.o # Green function database $O/green_function_detect.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_stf.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_locate.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index 17d6f37d3..039247aaa 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -73,6 +73,13 @@ subroutine setup_sources_receivers() endif call synchronize_all() + ! Green function database: locate elements containing GF locations + ! (must be called before xadj/adjncy are deallocated — needed for Stage 5 neighbor expansion) + if (GF_DATABASE_ENABLED) then + call gf_read_locations() + call gf_locate_elements() + endif + ! topography array no more needed if (TOPOGRAPHY) then if (allocated(ibathy_topo) ) deallocate(ibathy_topo) From e54f98a355d17fc6ba3801b897bcaaca6b5e5fc7 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sat, 9 May 2026 22:16:19 -0400 Subject: [PATCH 03/20] Add Stage 5: neighbor expansion for Green function database green_function_expand.F90 expands tagged elements by GF_NEIGHBOR_SHELLS layers using xadj/adjncy CSR adjacency (local) and ibool_interfaces exchange (cross-MPI). Shared gf_report_tagged_elements() gathers and prints per-rank element counts and IDs on rank 0. --- src/specfem3D/green_function_expand.F90 | 338 ++++++++++++++++++++++ src/specfem3D/green_function_locate.F90 | 120 +++++++- src/specfem3D/rules.mk | 2 + src/specfem3D/setup_sources_receivers.f90 | 1 + 4 files changed, 456 insertions(+), 5 deletions(-) create mode 100644 src/specfem3D/green_function_expand.F90 diff --git a/src/specfem3D/green_function_expand.F90 b/src/specfem3D/green_function_expand.F90 new file mode 100644 index 000000000..fe0008249 --- /dev/null +++ b/src/specfem3D/green_function_expand.F90 @@ -0,0 +1,338 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---- +!---- Green function database: expand tagged elements to include neighbor shells +!---- + + subroutine gf_expand_neighbors() + +! Expands the set of tagged elements by GF_NEIGHBOR_SHELLS layers of neighbors. +! +! Local expansion uses the xadj/adjncy CSR adjacency graph (within-rank). +! Cross-MPI expansion uses ibool_interfaces_crust_mantle to find elements +! on neighboring ranks that share nodes with tagged boundary elements. +! +! Must be called BEFORE xadj/adjncy are deallocated. + + use constants_solver, only: IMAIN,NGLLX,NGLLY,NGLLZ,NPROCTOT_VAL,itag + + use shared_parameters, only: GF_NEIGHBOR_SHELLS + + use specfem_par, only: & + myrank, & + nspec => NSPEC_CRUST_MANTLE, & + nglob => NGLOB_CRUST_MANTLE, & + xadj, adjncy + + use specfem_par, only: & + num_interfaces_crust_mantle, & + max_nibool_interfaces_cm, & + my_neighbors_crust_mantle, & + nibool_interfaces_crust_mantle, & + ibool_interfaces_crust_mantle + + use specfem_par_crustmantle, only: & + ibool => ibool_crust_mantle + + use green_function_par, only: & + gf_nelem_local, gf_local_elements + + implicit none + + ! local parameters + integer :: ishell, ispec, j, ier + integer :: nelem_after + + ! boolean mask for tagged elements (O(n) deduplication) + logical, allocatable, dimension(:) :: ispec_tagged + + ! nothing to do if no expansion requested or no local elements + if (GF_NEIGHBOR_SHELLS <= 0) then + if (myrank == 0) then + write(IMAIN,*) 'Green function database: GF_NEIGHBOR_SHELLS = 0, no expansion' + write(IMAIN,*) + call flush_IMAIN() + endif + return + endif + + if (myrank == 0) then + write(IMAIN,*) + write(IMAIN,*) '********************' + write(IMAIN,*) ' Green function database: expanding neighbor shells' + write(IMAIN,*) '********************' + write(IMAIN,*) + write(IMAIN,*) 'GF_NEIGHBOR_SHELLS = ',GF_NEIGHBOR_SHELLS + call flush_IMAIN() + endif + + ! build boolean mask from current tagged elements + allocate(ispec_tagged(nspec),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating ispec_tagged in gf_expand_neighbors') + ispec_tagged(:) = .false. + + do j = 1, gf_nelem_local + ispec_tagged(gf_local_elements(j)) = .true. + enddo + + ! expand by GF_NEIGHBOR_SHELLS layers + do ishell = 1, GF_NEIGHBOR_SHELLS + + ! --- local expansion using xadj/adjncy --- + call gf_expand_local(nspec,ispec_tagged,xadj,adjncy) + + ! --- cross-MPI expansion --- + if (NPROCTOT_VAL > 1) then + call gf_expand_cross_mpi(nspec,nglob,ispec_tagged,ibool, & + num_interfaces_crust_mantle, & + max_nibool_interfaces_cm, & + my_neighbors_crust_mantle, & + nibool_interfaces_crust_mantle, & + ibool_interfaces_crust_mantle) + endif + + enddo + + ! rebuild gf_local_elements from updated mask + nelem_after = count(ispec_tagged) + + ! reallocate if size changed + if (nelem_after /= gf_nelem_local) then + if (allocated(gf_local_elements)) deallocate(gf_local_elements) + gf_nelem_local = nelem_after + if (gf_nelem_local > 0) then + allocate(gf_local_elements(gf_nelem_local),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating gf_local_elements after expansion') + j = 0 + do ispec = 1, nspec + if (ispec_tagged(ispec)) then + j = j + 1 + gf_local_elements(j) = ispec + endif + enddo + endif + endif + + deallocate(ispec_tagged) + + ! gather and report per-rank element counts and IDs on rank 0 + call gf_report_tagged_elements('after neighbor expansion') + + end subroutine gf_expand_neighbors + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_expand_local(nspec,ispec_tagged,xadj,adjncy) + +! Expands tagged elements by one layer using the local xadj/adjncy CSR adjacency. +! Uses a snapshot of currently tagged elements to avoid chain expansion within +! a single layer. + + implicit none + + integer, intent(in) :: nspec + logical, dimension(nspec), intent(inout) :: ispec_tagged + integer, dimension(nspec+1), intent(in) :: xadj + integer, dimension(*), intent(in) :: adjncy + + ! local parameters + integer :: ispec, iadj, ineighbor + + ! snapshot: which elements are tagged at the START of this layer + logical, allocatable :: tagged_snapshot(:) + integer :: ier + + allocate(tagged_snapshot(nspec),stat=ier) + if (ier /= 0) stop 'Error allocating tagged_snapshot' + tagged_snapshot(:) = ispec_tagged(:) + + do ispec = 1, nspec + if (.not. tagged_snapshot(ispec)) cycle + + ! expand to all neighbors of this element + do iadj = xadj(ispec) + 1, xadj(ispec + 1) + ineighbor = adjncy(iadj) + if (ineighbor >= 1 .and. ineighbor <= nspec) then + ispec_tagged(ineighbor) = .true. + endif + enddo + enddo + + deallocate(tagged_snapshot) + + end subroutine gf_expand_local + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_expand_cross_mpi(nspec,nglob,ispec_tagged,ibool, & + num_interfaces,max_nibool_interfaces, & + my_neighbors,nibool_interfaces, & + ibool_interfaces) + +! Cross-MPI neighbor expansion. +! +! For tagged boundary elements, finds which of their GLL nodes are shared +! with neighboring MPI ranks via ibool_interfaces. Sends those shared node +! indices to the neighbor rank, which then tags any of its elements that +! contain those nodes. +! +! Uses send_i/recv_i with careful ordering (lower rank sends first) to +! avoid deadlock. + + use constants_solver, only: NGLLX,NGLLY,NGLLZ,itag + + use specfem_par, only: myrank + + implicit none + + integer, intent(in) :: nspec, nglob + logical, dimension(nspec), intent(inout) :: ispec_tagged + integer, dimension(NGLLX,NGLLY,NGLLZ,nspec), intent(in) :: ibool + integer, intent(in) :: num_interfaces, max_nibool_interfaces + integer, dimension(num_interfaces), intent(in) :: my_neighbors, nibool_interfaces + integer, dimension(max_nibool_interfaces,num_interfaces), intent(in) :: ibool_interfaces + + ! local parameters + integer :: iinterface, ipoin, iglob, ispec, i, j, k, ier + integer :: nsend, nrecv, neighbor_rank + integer :: ntagged_on_interface + + ! node-is-tagged lookup (built from tagged elements) + logical, allocatable :: node_tagged(:) + + ! send/recv buffers for shared node flags + integer, allocatable :: send_flags(:), recv_flags(:) + + ! reverse lookup: node -> element mapping for boundary nodes only + ! We use ibool to find which elements contain received nodes + logical, allocatable :: node_is_shared(:) + + ! nothing to do with single process + if (num_interfaces == 0) return + + ! build node_tagged: mark all GLL nodes of tagged elements + allocate(node_tagged(nglob),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating node_tagged') + node_tagged(:) = .false. + + do ispec = 1, nspec + if (.not. ispec_tagged(ispec)) cycle + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + node_tagged(ibool(i,j,k,ispec)) = .true. + enddo + enddo + enddo + enddo + + ! for each MPI interface, exchange flags indicating which shared nodes + ! belong to tagged elements + do iinterface = 1, num_interfaces + neighbor_rank = my_neighbors(iinterface) + nsend = nibool_interfaces(iinterface) + nrecv = nsend ! symmetric interface + + allocate(send_flags(nsend), recv_flags(nrecv), stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating MPI exchange buffers') + + ! pack: 1 if this shared node belongs to a tagged element, 0 otherwise + do ipoin = 1, nsend + iglob = ibool_interfaces(ipoin, iinterface) + if (node_tagged(iglob)) then + send_flags(ipoin) = 1 + else + send_flags(ipoin) = 0 + endif + enddo + + ! exchange using send/recv with ordering to avoid deadlock + if (myrank < neighbor_rank) then + call send_i(send_flags, nsend, neighbor_rank, itag) + call recv_i(recv_flags, nrecv, neighbor_rank, itag) + else + call recv_i(recv_flags, nrecv, neighbor_rank, itag) + call send_i(send_flags, nsend, neighbor_rank, itag) + endif + + ! process received flags: for shared nodes flagged by the neighbor, + ! find which local elements contain those nodes and tag them + ntagged_on_interface = 0 + do ipoin = 1, nrecv + if (recv_flags(ipoin) == 1) then + iglob = ibool_interfaces(ipoin, iinterface) + ntagged_on_interface = ntagged_on_interface + 1 + ! mark this node so we can find its elements + node_tagged(iglob) = .true. + endif + enddo + + ! if any nodes were flagged, find elements containing them + if (ntagged_on_interface > 0) then + ! build a quick set of received flagged node IDs + allocate(node_is_shared(nglob), stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating node_is_shared') + node_is_shared(:) = .false. + + do ipoin = 1, nrecv + if (recv_flags(ipoin) == 1) then + iglob = ibool_interfaces(ipoin, iinterface) + node_is_shared(iglob) = .true. + endif + enddo + + ! scan all elements: if any corner node is in the received set, tag it + do ispec = 1, nspec + if (ispec_tagged(ispec)) cycle ! already tagged + + ! check 8 corner nodes (sufficient for adjacency — same as xadj/adjncy) + if (node_is_shared(ibool(1,1,1,ispec)) .or. & + node_is_shared(ibool(NGLLX,1,1,ispec)) .or. & + node_is_shared(ibool(NGLLX,NGLLY,1,ispec)) .or. & + node_is_shared(ibool(1,NGLLY,1,ispec)) .or. & + node_is_shared(ibool(1,1,NGLLZ,ispec)) .or. & + node_is_shared(ibool(NGLLX,1,NGLLZ,ispec)) .or. & + node_is_shared(ibool(NGLLX,NGLLY,NGLLZ,ispec)) .or. & + node_is_shared(ibool(1,NGLLY,NGLLZ,ispec))) then + ispec_tagged(ispec) = .true. + endif + enddo + + deallocate(node_is_shared) + endif + + deallocate(send_flags, recv_flags) + enddo + + deallocate(node_tagged) + + end subroutine gf_expand_cross_mpi diff --git a/src/specfem3D/green_function_locate.F90 b/src/specfem3D/green_function_locate.F90 index 388248450..a2fe64e5b 100644 --- a/src/specfem3D/green_function_locate.F90 +++ b/src/specfem3D/green_function_locate.F90 @@ -155,7 +155,7 @@ subroutine gf_locate_elements() use constants_solver, only: & ELLIPTICITY_VAL,NDIM,HUGEVAL,IMAIN, & DEGREES_TO_RADIANS,R_UNIT_SPHERE, & - nrec_SUBSET_MAX + nrec_SUBSET_MAX,NPROCTOT_VAL use shared_parameters, only: R_PLANET @@ -396,12 +396,122 @@ subroutine gf_locate_elements() deallocate(ispec_tagged) - ! report per-rank element counts + ! gather and report per-rank element counts and IDs on rank 0 + call gf_report_tagged_elements('initial tagging') + + end subroutine gf_locate_elements + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_report_tagged_elements(label) + +! Gathers per-rank tagged element counts and element IDs on rank 0 and prints them. +! Called after initial tagging (Stage 4) and after neighbor expansion (Stage 5). + + use constants_solver, only: IMAIN,NPROCTOT_VAL + + use specfem_par, only: myrank + + use green_function_par, only: gf_nelem_local,gf_local_elements + + implicit none + + character(len=*), intent(in) :: label + + ! number of element IDs to print per line + integer, parameter :: ELEMS_PER_LINE = 10 + + ! local parameters + integer :: irank, ier, j, nelem_total + integer, allocatable :: counts_all(:) ! element count per rank (on rank 0) + integer, allocatable :: offsets(:) ! offsets for gatherv + integer, allocatable :: elems_all(:) ! all element IDs gathered (on rank 0) + integer :: sendcount_dummy(1) + + if (myrank == 0) then + allocate(counts_all(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating counts_all') + else + allocate(counts_all(1),stat=ier) + endif + + ! gather element counts from all ranks to rank 0 + sendcount_dummy(1) = gf_nelem_local + call gather_all_i(sendcount_dummy, 1, counts_all, 1, NPROCTOT_VAL) + + ! gather element IDs using gatherv + if (myrank == 0) then + nelem_total = sum(counts_all) + + allocate(offsets(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating offsets') + + ! compute offsets for gatherv + offsets(0) = 0 + do irank = 1, NPROCTOT_VAL - 1 + offsets(irank) = offsets(irank - 1) + counts_all(irank - 1) + enddo + + if (nelem_total > 0) then + allocate(elems_all(nelem_total),stat=ier) + else + allocate(elems_all(1),stat=ier) + endif + if (ier /= 0) call exit_MPI(myrank,'Error allocating elems_all') + else + allocate(offsets(1),stat=ier) + allocate(elems_all(1),stat=ier) + nelem_total = 0 + endif + + ! gatherv: each rank sends its gf_local_elements + if (gf_nelem_local > 0) then + call gatherv_all_i(gf_local_elements, gf_nelem_local, & + elems_all, counts_all, offsets, & + max(nelem_total,1), NPROCTOT_VAL) + else + ! send a dummy (zero-length send is handled by MPI) + call gatherv_all_i(sendcount_dummy, 0, & + elems_all, counts_all, offsets, & + max(nelem_total,1), NPROCTOT_VAL) + endif + + ! print on rank 0 if (myrank == 0) then - write(IMAIN,*) 'Green function database: element tagging summary' - write(IMAIN,*) ' Rank 0 has ',gf_nelem_local,' unique tagged elements' + write(IMAIN,*) + write(IMAIN,*) 'Green function database: tagged elements (',trim(label),')' + write(IMAIN,*) ' Total elements across all ranks: ',nelem_total + write(IMAIN,*) + + do irank = 0, NPROCTOT_VAL - 1 + write(IMAIN,'(a,i6,a,i6,a)') & + ' Rank ',irank,': ',counts_all(irank),' elements' + if (counts_all(irank) > 0) then + write(IMAIN,'(a)',advance='no') ' [' + do j = 1, counts_all(irank) + if (j > 1) write(IMAIN,'(a)',advance='no') ', ' + ! line break every ELEMS_PER_LINE elements + if (j > 1 .and. mod(j - 1, ELEMS_PER_LINE) == 0) then + write(IMAIN,*) + write(IMAIN,'(a)',advance='no') ' ' + endif + write(IMAIN,'(i0)',advance='no') elems_all(offsets(irank) + j) + enddo + write(IMAIN,'(a)') ']' + endif + enddo + write(IMAIN,*) call flush_IMAIN() + + deallocate(offsets) + else + deallocate(offsets) endif - end subroutine gf_locate_elements + deallocate(counts_all) + deallocate(elems_all) + + end subroutine gf_report_tagged_elements diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index 764e99e49..74bb95510 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -90,6 +90,7 @@ specfem3D_SOLVER_OBJECTS += \ $O/green_function_detect.solverstatic.o \ $O/green_function_stf.solverstatic.o \ $O/green_function_locate.solverstatic.o \ + $O/green_function_expand.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ $O/iterate_time_undoatt.solverstatic.o \ @@ -465,6 +466,7 @@ $O/prepare_gravity.solverstatic.o: $O/SIEM_math_library.shared.o $O/green_function_detect.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_stf.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_locate.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_expand.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index 039247aaa..3b5ff1f55 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -78,6 +78,7 @@ subroutine setup_sources_receivers() if (GF_DATABASE_ENABLED) then call gf_read_locations() call gf_locate_elements() + call gf_expand_neighbors() endif ! topography array no more needed From b6b37ca057ccfaed5e9b34cf77969f1cc5167be1 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sat, 9 May 2026 22:49:15 -0400 Subject: [PATCH 04/20] Add Stage 6: Morton coding for Green function database Compute 63-bit Morton codes from element center GLL node coordinates (21-bit quantization, bit-interleaved via spread_bits). Create per-element directories under GF_DATABASE_PATH/elements/{morton_hex}/. Write centroids.bin (binary, 32 bytes/record: int64 morton + 3x float64 centroid) and manifest.csv gathered from all MPI ranks via send_cr/recv_cr. New files: - green_function_morton.F90: gf_spread_bits, gf_morton_encode, gf_compute_morton_codes, gf_create_directories, gf_write_manifest - prepare_green_function_storage.f90: central wrapper consolidating all GF preparation (STF + Morton + dirs + manifest) into a single call from prepare_timerun - utils/green_function/verify_morton.py: cross-validates Fortran Morton codes against Python reimplementation - utils/green_function/decode_morton.py: decodes Morton hex back to approximate coordinates (lat/lon/r) Modified files: - green_function_par.F90: add gf_morton_codes, gf_morton_hex, gf_center_xyz - prepare_timerun.F90: replace two GF calls with single call prepare_green_function_storage() - green_function_stf.F90: remove dead prepare_greenfunction_stf wrapper - rules.mk: add green_function_morton and prepare_green_function_storage Tested with 98 elements on 4 MPI procs. All 98 Morton codes verified bit-for-bit against Python implementation. Max quantization error 1.4e-6. --- src/specfem3D/green_function_morton.F90 | 403 ++++++++++++++++++ src/specfem3D/green_function_par.F90 | 5 + src/specfem3D/green_function_stf.F90 | 15 - .../prepare_green_function_storage.f90 | 51 +++ src/specfem3D/prepare_timerun.F90 | 5 +- src/specfem3D/rules.mk | 3 + utils/green_function/decode_morton.py | 118 +++++ utils/green_function/validate_stf.py | 144 +++++++ utils/green_function/verify_morton.py | 105 +++++ 9 files changed, 832 insertions(+), 17 deletions(-) create mode 100644 src/specfem3D/green_function_morton.F90 create mode 100644 src/specfem3D/prepare_green_function_storage.f90 create mode 100644 utils/green_function/decode_morton.py create mode 100644 utils/green_function/validate_stf.py create mode 100644 utils/green_function/verify_morton.py diff --git a/src/specfem3D/green_function_morton.F90 b/src/specfem3D/green_function_morton.F90 new file mode 100644 index 000000000..b8008be24 --- /dev/null +++ b/src/specfem3D/green_function_morton.F90 @@ -0,0 +1,403 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---- +!---- Green function database: Morton code computation and manifest writing +!---- + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_compute_morton_codes() + +! Computes center coordinates and Morton codes for all tagged local elements. +! Must be called while xstore/ystore/zstore_crust_mantle are still allocated. + + use constants, only: CUSTOM_REAL,MIDX,MIDY,MIDZ,IMAIN + + use specfem_par, only: myrank + + use specfem_par_crustmantle, only: & + ibool => ibool_crust_mantle, & + xstore => xstore_crust_mantle, & + ystore => ystore_crust_mantle, & + zstore => zstore_crust_mantle + + use green_function_par, only: & + gf_nelem_local, gf_local_elements, & + gf_morton_codes, gf_morton_hex, gf_center_xyz + + implicit none + + ! external functions + integer(8), external :: gf_morton_encode + + ! local parameters + integer :: i, ispec, iglob_center, ier + + if (gf_nelem_local == 0) return + + ! allocate arrays + allocate(gf_center_xyz(3, gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_center_xyz') + + allocate(gf_morton_codes(gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_codes') + + allocate(gf_morton_hex(gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_hex') + + ! extract center GLL node coordinates and compute Morton codes + do i = 1, gf_nelem_local + ispec = gf_local_elements(i) + iglob_center = ibool(MIDX, MIDY, MIDZ, ispec) + + gf_center_xyz(1, i) = xstore(iglob_center) + gf_center_xyz(2, i) = ystore(iglob_center) + gf_center_xyz(3, i) = zstore(iglob_center) + + ! gf_morton_encode is a standalone function defined below + gf_morton_codes(i) = gf_morton_encode( & + gf_center_xyz(1, i), gf_center_xyz(2, i), gf_center_xyz(3, i)) + + write(gf_morton_hex(i), '(Z16.16)') gf_morton_codes(i) + enddo + + ! report on rank 0 + call gf_report_morton_codes() + + end subroutine gf_compute_morton_codes + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_report_morton_codes() + +! Gathers element counts from all ranks and prints Morton code info on rank 0. + + use constants, only: IMAIN,CUSTOM_REAL + use constants_solver, only: NPROCTOT_VAL + + use specfem_par, only: myrank + + use green_function_par, only: & + gf_nelem_local, gf_morton_hex, gf_center_xyz + + implicit none + + integer :: irank, ier, j, nelem_total + integer, allocatable :: counts_all(:) + integer :: sendcount_dummy(1) + + ! gather counts on rank 0 + allocate(counts_all(0:NPROCTOT_VAL-1), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating counts_all in gf_report_morton_codes') + + sendcount_dummy(1) = gf_nelem_local + call gather_all_i(sendcount_dummy, 1, counts_all, 1, NPROCTOT_VAL) + + if (myrank == 0) then + nelem_total = sum(counts_all) + + write(IMAIN,*) + write(IMAIN,*) '********************' + write(IMAIN,*) ' Green function database: Morton codes' + write(IMAIN,*) '********************' + write(IMAIN,*) + write(IMAIN,*) 'Total tagged elements across all ranks: ', nelem_total + write(IMAIN,*) + + do irank = 0, NPROCTOT_VAL - 1 + write(IMAIN,'(a,i6,a,i6,a)') ' Rank ', irank, ': ', counts_all(irank), ' elements' + enddo + + write(IMAIN,*) + write(IMAIN,*) 'Rank 0 Morton codes (sample):' + do j = 1, min(gf_nelem_local, 10) + write(IMAIN,'(a,i6,a,a,a,3(f12.6))') ' elem ', j, ': ', & + gf_morton_hex(j), ' center=', & + gf_center_xyz(1,j), gf_center_xyz(2,j), gf_center_xyz(3,j) + enddo + if (gf_nelem_local > 10) then + write(IMAIN,'(a,i6,a)') ' ... (', gf_nelem_local - 10, ' more)' + endif + write(IMAIN,*) + call flush_IMAIN() + endif + + deallocate(counts_all) + + end subroutine gf_report_morton_codes + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_create_directories() + +! Creates the database directory structure: +! {GF_DATABASE_PATH}/elements/{morton_hex}/ +! Each rank creates directories for its own elements. + + use constants, only: IMAIN,MAX_STRING_LEN + + use shared_parameters, only: GF_DATABASE_PATH + + use specfem_par, only: myrank + + use green_function_par, only: & + gf_nelem_local, gf_morton_hex + + implicit none + + integer :: i + character(len=MAX_STRING_LEN) :: dirpath + + ! rank 0 creates base directories + if (myrank == 0) then + call system('mkdir -p ' // trim(GF_DATABASE_PATH) // '/elements') + + write(IMAIN,*) 'Green function database: creating directories' + write(IMAIN,*) ' base path: ', trim(GF_DATABASE_PATH) + call flush_IMAIN() + endif + + ! synchronize so base directory exists before ranks create subdirectories + call synchronize_all() + + ! each rank creates directories for its local elements + do i = 1, gf_nelem_local + dirpath = trim(GF_DATABASE_PATH) // '/elements/' // gf_morton_hex(i) + call system('mkdir -p ' // trim(dirpath)) + enddo + + call synchronize_all() + + if (myrank == 0) then + write(IMAIN,*) 'Green function database: directories created' + write(IMAIN,*) + call flush_IMAIN() + endif + + end subroutine gf_create_directories + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_write_manifest() + +! Writes manifest files: +! {GF_DATABASE_PATH}/elements/centroids.bin — binary (morton_code + cx,cy,cz as float64) +! {GF_DATABASE_PATH}/elements/manifest.csv — human-readable +! +! Only rank 0 writes. Center coordinates (CUSTOM_REAL) are gathered from all +! ranks via send_cr/recv_cr. Rank 0 recomputes Morton codes from the gathered +! float values to avoid needing int(8) MPI transfers. + + use constants, only: IMAIN,MAX_STRING_LEN,CUSTOM_REAL + use constants_solver, only: NPROCTOT_VAL + + use shared_parameters, only: GF_DATABASE_PATH + + use specfem_par, only: myrank + + use green_function_par, only: & + gf_nelem_local, gf_center_xyz + + implicit none + + ! external functions + integer(8), external :: gf_morton_encode + + integer :: irank, i, j, ier, nelem_total + integer, allocatable :: counts_all(:), offsets(:) + integer :: sendcount_dummy(1) + + ! gathered data on rank 0 + real(kind=CUSTOM_REAL), allocatable :: center_all(:,:) + integer(8) :: morton_val + character(len=16) :: hex_str + + ! file I/O + character(len=MAX_STRING_LEN) :: filepath_bin, filepath_csv + integer, parameter :: IUNIT_BIN = 71 + integer, parameter :: IUNIT_CSV = 72 + + ! gather counts + allocate(counts_all(0:NPROCTOT_VAL-1), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating counts_all in gf_write_manifest') + sendcount_dummy(1) = gf_nelem_local + call gather_all_i(sendcount_dummy, 1, counts_all, 1, NPROCTOT_VAL) + + if (myrank == 0) then + ! build offsets + allocate(offsets(0:NPROCTOT_VAL-1), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating offsets in gf_write_manifest') + offsets(0) = 0 + do irank = 1, NPROCTOT_VAL - 1 + offsets(irank) = offsets(irank - 1) + counts_all(irank - 1) + enddo + nelem_total = offsets(NPROCTOT_VAL - 1) + counts_all(NPROCTOT_VAL - 1) + else + nelem_total = 0 + endif + + ! gather center coordinates on rank 0 + if (myrank == 0) then + allocate(center_all(3, nelem_total), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating center_all') + + ! copy rank 0 data + do i = 1, counts_all(0) + center_all(1:3, i) = gf_center_xyz(1:3, i) + enddo + + ! receive from other ranks + do irank = 1, NPROCTOT_VAL - 1 + if (counts_all(irank) > 0) then + j = offsets(irank) + 1 + call recv_cr(center_all(1, j), 3 * counts_all(irank), irank, 100) + endif + enddo + else + ! non-zero ranks send their data + if (gf_nelem_local > 0) then + call send_cr(gf_center_xyz, 3 * gf_nelem_local, 0, 100) + endif + endif + + ! rank 0 writes manifest files + if (myrank == 0) then + + filepath_bin = trim(GF_DATABASE_PATH) // '/elements/centroids.bin' + filepath_csv = trim(GF_DATABASE_PATH) // '/elements/manifest.csv' + + ! write binary manifest + open(unit=IUNIT_BIN, file=trim(filepath_bin), form='unformatted', & + access='stream', status='replace', iostat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error opening centroids.bin') + + ! write CSV manifest + open(unit=IUNIT_CSV, file=trim(filepath_csv), form='formatted', & + status='replace', iostat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error opening manifest.csv') + write(IUNIT_CSV, '(a)') 'morton_hex,cx,cy,cz' + + do i = 1, nelem_total + ! recompute Morton code from gathered float coordinates + morton_val = gf_morton_encode(center_all(1,i), center_all(2,i), center_all(3,i)) + write(hex_str, '(Z16.16)') morton_val + + ! binary: int64 morton + 3x float64 centroid = 32 bytes per record + write(IUNIT_BIN) morton_val, dble(center_all(1,i)), & + dble(center_all(2,i)), dble(center_all(3,i)) + + ! CSV + write(IUNIT_CSV, '(a,a,ES22.14,a,ES22.14,a,ES22.14)') & + hex_str, ',', dble(center_all(1,i)), ',', dble(center_all(2,i)), & + ',', dble(center_all(3,i)) + enddo + + close(IUNIT_BIN) + close(IUNIT_CSV) + + write(IMAIN,*) 'Green function database: manifest files written' + write(IMAIN,*) ' centroids.bin: ', nelem_total, ' records (32 bytes each)' + write(IMAIN,*) ' manifest.csv: ', nelem_total, ' entries' + write(IMAIN,*) + call flush_IMAIN() + + deallocate(center_all, offsets) + endif + + deallocate(counts_all) + + end subroutine gf_write_manifest + +! +!------------------------------------------------------------------------------------------------- +! + + function gf_morton_encode(cx, cy, cz) result(morton) + +! Standalone Morton encoder (not inside a contains block) so it can be +! called from gf_write_manifest. + + use constants, only: CUSTOM_REAL + + implicit none + + real(kind=CUSTOM_REAL), intent(in) :: cx, cy, cz + integer(8) :: morton + integer(8) :: ix, iy, iz + + ! external functions + integer(8), external :: gf_spread_bits + + integer, parameter :: MORTON_BITS = 21 + integer, parameter :: MORTON_BINS = 2**MORTON_BITS ! 2,097,152 + + ! quantize from [-1, 1] to [0, MORTON_BINS - 1] + ix = int((real(cx) + 1.0) * 0.5 * real(MORTON_BINS - 1), 8) + iy = int((real(cy) + 1.0) * 0.5 * real(MORTON_BINS - 1), 8) + iz = int((real(cz) + 1.0) * 0.5 * real(MORTON_BINS - 1), 8) + + ! clamp to valid range + ix = max(int(0, 8), min(ix, int(MORTON_BINS - 1, 8))) + iy = max(int(0, 8), min(iy, int(MORTON_BINS - 1, 8))) + iz = max(int(0, 8), min(iz, int(MORTON_BINS - 1, 8))) + + ! bit-interleave: x in bits 0,3,6,...; y in bits 1,4,7,...; z in bits 2,5,8,... + morton = ior(ior(gf_spread_bits(ix), ishft(gf_spread_bits(iy), 1)), & + ishft(gf_spread_bits(iz), 2)) + + end function gf_morton_encode + +! +!------------------------------------------------------------------------------------------------- +! + + function gf_spread_bits(v) result(spread) + +! Spread 21 low bits of v into every-third-bit positions. +! Standalone version callable from gf_morton_encode and gf_compute_morton_codes. + + implicit none + integer(8), intent(in) :: v + integer(8) :: spread + + spread = v + spread = iand(ior(spread, ishft(spread, 32)), int(Z'001F00000000FFFF', 8)) + spread = iand(ior(spread, ishft(spread, 16)), int(Z'001F0000FF0000FF', 8)) + spread = iand(ior(spread, ishft(spread, 8)), int(Z'100F00F00F00F00F', 8)) + spread = iand(ior(spread, ishft(spread, 4)), int(Z'10C30C30C30C30C3', 8)) + spread = iand(ior(spread, ishft(spread, 2)), int(Z'1249249249249249', 8)) + + end function gf_spread_bits diff --git a/src/specfem3D/green_function_par.F90 b/src/specfem3D/green_function_par.F90 index a7622286f..88b6ae6e7 100644 --- a/src/specfem3D/green_function_par.F90 +++ b/src/specfem3D/green_function_par.F90 @@ -51,4 +51,9 @@ module green_function_par integer :: gf_nelem_local = 0 ! unique tagged elements on this rank integer, dimension(:), allocatable :: gf_local_elements ! unique local element indices + ! Morton codes (Stage 6) + integer(8), dimension(:), allocatable :: gf_morton_codes ! Morton code per local element + character(len=16), dimension(:), allocatable :: gf_morton_hex ! hex string per local element + real(kind=CUSTOM_REAL), dimension(:,:), allocatable :: gf_center_xyz ! (3, gf_nelem_local) center coords + end module green_function_par diff --git a/src/specfem3D/green_function_stf.F90 b/src/specfem3D/green_function_stf.F90 index c94840776..4b4449079 100644 --- a/src/specfem3D/green_function_stf.F90 +++ b/src/specfem3D/green_function_stf.F90 @@ -33,21 +33,6 @@ ! filtering for zero-phase response (equivalent to scipy filtfilt). !---------------------------------------------------------------------- - subroutine prepare_greenfunction_stf() - -! Wrapper called from prepare_timerun(). -! Checks if GF database is enabled and computes the STF. - - use shared_parameters, only: GF_DATABASE_ENABLED - - implicit none - - if (.not. GF_DATABASE_ENABLED) return - - call gf_compute_stf() - - end subroutine prepare_greenfunction_stf - ! !---------------------------------------------------------------------- ! diff --git a/src/specfem3D/prepare_green_function_storage.f90 b/src/specfem3D/prepare_green_function_storage.f90 new file mode 100644 index 000000000..56784bc90 --- /dev/null +++ b/src/specfem3D/prepare_green_function_storage.f90 @@ -0,0 +1,51 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine prepare_green_function_storage() + +! Central wrapper for all Green function database preparation steps. +! Called from prepare_timerun() before coordinate arrays are deallocated. + + use shared_parameters, only: GF_DATABASE_ENABLED + + implicit none + + if (.not. GF_DATABASE_ENABLED) return + + ! precompute Butterworth-filtered STF + call gf_compute_stf() + + ! compute Morton codes from element center coordinates + call gf_compute_morton_codes() + + ! create directory structure + call gf_create_directories() + + ! write centroids.bin and manifest.csv + call gf_write_manifest() + + end subroutine prepare_green_function_storage diff --git a/src/specfem3D/prepare_timerun.F90 b/src/specfem3D/prepare_timerun.F90 index 75a20d354..962623815 100644 --- a/src/specfem3D/prepare_timerun.F90 +++ b/src/specfem3D/prepare_timerun.F90 @@ -94,8 +94,9 @@ subroutine prepare_timerun() ! prepares VTK window visualization call prepare_vtk_window() - ! Green function database: precompute Butterworth-filtered STF - call prepare_greenfunction_stf() + ! Green function database: STF, Morton codes, directories, manifest + ! (must be called before xstore/ystore/zstore_crust_mantle are deallocated) + call prepare_green_function_storage() ! optimizes array memory layout for better performance call prepare_optimized_arrays() diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index 74bb95510..a3abef813 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -91,6 +91,8 @@ specfem3D_SOLVER_OBJECTS += \ $O/green_function_stf.solverstatic.o \ $O/green_function_locate.solverstatic.o \ $O/green_function_expand.solverstatic.o \ + $O/green_function_morton.solverstatic.o \ + $O/prepare_green_function_storage.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ $O/iterate_time_undoatt.solverstatic.o \ @@ -467,6 +469,7 @@ $O/green_function_detect.solverstatic.o: $O/green_function_par.solverstatic_modu $O/green_function_stf.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_locate.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_expand.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_morton.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here diff --git a/utils/green_function/decode_morton.py b/utils/green_function/decode_morton.py new file mode 100644 index 000000000..6f91d3d87 --- /dev/null +++ b/utils/green_function/decode_morton.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Decode Morton codes back to approximate coordinates. + +Usage: + python decode_morton.py [morton_hex ...] + python decode_morton.py --file path/to/centroids.bin + +Examples: + python decode_morton.py 113B5BC4F83F7641 + python decode_morton.py --file DB_TESTING/db_gen/OUTPUT_FILES/gf_database/elements/centroids.bin +""" + +import struct +import sys +import numpy as np + + +MORTON_BITS = 21 +MORTON_BINS = 2**MORTON_BITS # 2,097,152 + + +def compact_bits(v): + """Reverse of spread_bits: extract every-third bit into contiguous low bits.""" + v = v & 0x1249249249249249 + v = (v | (v >> 2)) & 0x10C30C30C30C30C3 + v = (v | (v >> 4)) & 0x100F00F00F00F00F + v = (v | (v >> 8)) & 0x001F0000FF0000FF + v = (v | (v >> 16)) & 0x001F00000000FFFF + v = (v | (v >> 32)) & 0x00000000001FFFFF + return v + + +def morton_decode_3d(morton): + """Decode a 63-bit Morton code back to approximate (x, y, z) in [-1, 1].""" + ix = compact_bits(morton) + iy = compact_bits(morton >> 1) + iz = compact_bits(morton >> 2) + + # map from [0, MORTON_BINS - 1] back to [-1, 1] + x = ix / (MORTON_BINS - 1) * 2.0 - 1.0 + y = iy / (MORTON_BINS - 1) * 2.0 - 1.0 + z = iz / (MORTON_BINS - 1) * 2.0 - 1.0 + + return x, y, z + + +def xyz_to_latlon(x, y, z): + """Convert Cartesian unit-sphere coordinates to latitude/longitude (degrees). + + The coordinates are normalized to the unit sphere (r ~ 1 for surface elements, + smaller for deeper elements). + """ + r = np.sqrt(x**2 + y**2 + z**2) + lat = np.degrees(np.arcsin(z / r)) if r > 0 else 0.0 + lon = np.degrees(np.arctan2(y, x)) + return lat, lon, r + + +def decode_hex(hex_str): + """Decode a Morton hex string and print coordinates.""" + morton = int(hex_str, 16) + x, y, z = morton_decode_3d(morton) + lat, lon, r = xyz_to_latlon(x, y, z) + print(f" Morton: {hex_str}") + print(f" Cartesian: x={x:.6f} y={y:.6f} z={z:.6f}") + print(f" Spherical: lat={lat:.4f} lon={lon:.4f} r={r:.6f}") + print() + return x, y, z + + +def decode_centroids_file(filepath): + """Decode all entries in a centroids.bin file and compare with stored coords.""" + with open(filepath, "rb") as f: + data = f.read() + + n_records = len(data) // 32 + print(f"Decoding {n_records} records from {filepath}\n") + print(f"{'Morton hex':>20s} {'dx':>10s} {'dy':>10s} {'dz':>10s}" + f" {'lat':>8s} {'lon':>9s} {'r':>8s}") + print("-" * 85) + + max_err = 0.0 + for i in range(n_records): + offset = i * 32 + morton_stored, cx, cy, cz = struct.unpack(" 0 else float('inf') + + print(f'Comparison:') + print(f' max |diff| unfiltered: {diff_unfiltered:.2e}') + print(f' max |diff| filtered: {diff_filtered:.2e}') + print(f' peak Fortran: {peak_f90:.6f}') + print(f' peak Python: {peak_py:.6f}') + print(f' relative diff: {rel_diff:.2e}') + print() + + if rel_diff < 1e-6: + print('PASS: Fortran and Python STFs match to within 1e-6 relative error.') + elif rel_diff < 1e-3: + print('CLOSE: Fortran and Python STFs differ slightly (check plot).') + else: + print('FAIL: Significant difference between Fortran and Python STFs.') + + # plot + fig, axes = plt.subplots(3, 1, figsize=(10, 8), sharex=True) + + axes[0].plot(t, stf_unfiltered_f90, 'b-', label='Fortran', linewidth=1.5) + axes[0].plot(t, stf_unfiltered_py, 'r--', label='Python', linewidth=1.5) + axes[0].set_ylabel('Amplitude') + axes[0].set_title('Unfiltered Gaussian STF') + axes[0].legend() + axes[0].grid(True, alpha=0.3) + + axes[1].plot(t, stf_filtered_f90, 'b-', label='Fortran', linewidth=1.5) + axes[1].plot(t, stf_filtered_py, 'r--', label='Python', linewidth=1.5) + axes[1].set_ylabel('Amplitude') + axes[1].set_title(f'Butterworth-filtered STF (order={order}, fc={f_cutoff:.3f} Hz)') + axes[1].legend() + axes[1].grid(True, alpha=0.3) + + axes[2].plot(t, stf_filtered_f90 - stf_filtered_py, 'k-', linewidth=1.0) + axes[2].set_ylabel('Difference') + axes[2].set_xlabel('Time (s)') + axes[2].set_title('Fortran - Python') + axes[2].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig('stf_cross_validation.png', dpi=150) + print(f'\nPlot saved to stf_cross_validation.png') + + +if __name__ == '__main__': + main() diff --git a/utils/green_function/verify_morton.py b/utils/green_function/verify_morton.py new file mode 100644 index 000000000..f89eca9bf --- /dev/null +++ b/utils/green_function/verify_morton.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Verify Morton encoding in centroids.bin against a Python reimplementation. + +Usage: + python verify_morton.py [path/to/centroids.bin] + +Default path: DB_TESTING/db_gen/OUTPUT_FILES/gf_database/elements/centroids.bin +""" + +import struct +import sys +import numpy as np + + +def spread_bits(v): + """Spread 21 low bits of v into every-third-bit positions.""" + v = v & 0x1FFFFF # 21 bits + v = (v | (v << 32)) & 0x001F00000000FFFF + v = (v | (v << 16)) & 0x001F0000FF0000FF + v = (v | (v << 8)) & 0x100F00F00F00F00F + v = (v | (v << 4)) & 0x10C30C30C30C30C3 + v = (v | (v << 2)) & 0x1249249249249249 + return v + + +def morton_encode_3d(cx, cy, cz): + """Quantize float32 coordinates from [-1,1] to 21-bit integers and + bit-interleave into a 63-bit Morton code. + + Must match the Fortran implementation in green_function_morton.F90. + """ + cx32 = np.float32(cx) + cy32 = np.float32(cy) + cz32 = np.float32(cz) + MORTON_BINS = 2**21 + ix = int(np.float32((cx32 + np.float32(1.0)) * np.float32(0.5) * np.float32(MORTON_BINS - 1))) + iy = int(np.float32((cy32 + np.float32(1.0)) * np.float32(0.5) * np.float32(MORTON_BINS - 1))) + iz = int(np.float32((cz32 + np.float32(1.0)) * np.float32(0.5) * np.float32(MORTON_BINS - 1))) + ix = max(0, min(ix, MORTON_BINS - 1)) + iy = max(0, min(iy, MORTON_BINS - 1)) + iz = max(0, min(iz, MORTON_BINS - 1)) + return spread_bits(ix) | (spread_bits(iy) << 1) | (spread_bits(iz) << 2) + + +def test_corner_cases(): + """Test spread_bits and morton_encode_3d with known values.""" + print("Corner case tests:") + assert spread_bits(0) == 0, f"spread_bits(0) = {spread_bits(0)}" + print(f" spread_bits(0) = {spread_bits(0)} [OK]") + + assert spread_bits(1) == 1, f"spread_bits(1) = {spread_bits(1)}" + print(f" spread_bits(1) = {spread_bits(1)} [OK]") + + assert spread_bits(7) == 73, f"spread_bits(7) = {spread_bits(7)}" + print(f" spread_bits(7) = {spread_bits(7)} [OK]") + + m = morton_encode_3d(-1, -1, -1) + assert m == 0, f"morton(-1,-1,-1) = {m:016X}" + print(f" morton(-1,-1,-1) = {m:016X} [OK]") + + m = morton_encode_3d(1, 1, 1) + assert m == (1 << 63) - 1, f"morton(1,1,1) = {m:016X}" + print(f" morton( 1, 1, 1) = {m:016X} [OK]") + + print() + + +def verify_centroids(filepath): + """Read centroids.bin and verify each Morton code against Python encoding.""" + with open(filepath, "rb") as f: + data = f.read() + + n_records = len(data) // 32 + print(f"Records in centroids.bin: {n_records}") + + mismatches = 0 + for i in range(n_records): + offset = i * 32 + morton_stored, cx, cy, cz = struct.unpack(" 1: + filepath = sys.argv[1] + else: + filepath = "DB_TESTING/db_gen/OUTPUT_FILES/gf_database/elements/centroids.bin" + + verify_centroids(filepath) From 1b0cd3d653022a103b72ecdfc3b7483e7c8b1cfa Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sat, 9 May 2026 23:36:33 -0400 Subject: [PATCH 05/20] =?UTF-8?q?Stage=207:=20HDF5=20I/O=20=E2=80=94=20buf?= =?UTF-8?q?fered=20per-element=20displacement=20writing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add green_function_io.F90 with serial HDF5 file creation, chunked 6D datasets (force_comp, disp_comp, NGLLX, NGLLY, NGLLZ, nt_sub), and buffered hyperslab writing. Each MPI rank independently manages its local tagged elements. Files are create-or-open so three sequential force component runs (N/E/Z) fill the same dataset without overwriting. Buffer layout (3,5,5,5,bufsize,nelem) uses element-last ordering for contiguous memory access per element. Uses c_loc for rank-agnostic h5dwrite_f calls. Fill value 0 ensures unwritten components are zero. Hooked into iterate_time.F90 (subsampled extraction) and finalize_simulation.F90 (final flush + cleanup). There is still the issue of the metadata, that is topography etc. --- src/specfem3D/finalize_simulation.F90 | 3 + src/specfem3D/green_function_io.F90 | 542 ++++++++++++++++++ src/specfem3D/green_function_par.F90 | 9 + src/specfem3D/iterate_time.F90 | 3 + .../prepare_green_function_storage.f90 | 3 + src/specfem3D/rules.mk | 2 + 6 files changed, 562 insertions(+) create mode 100644 src/specfem3D/green_function_io.F90 diff --git a/src/specfem3D/finalize_simulation.F90 b/src/specfem3D/finalize_simulation.F90 index 323f7a490..9c359ed2a 100644 --- a/src/specfem3D/finalize_simulation.F90 +++ b/src/specfem3D/finalize_simulation.F90 @@ -69,6 +69,9 @@ subroutine finalize_simulation() call close_file_abs(9) endif + ! Green function database: flush remaining buffer and close HDF5 + call gf_finalize_hdf5() + ! save files to local disk or tape system if restart file call save_forward_arrays() diff --git a/src/specfem3D/green_function_io.F90 b/src/specfem3D/green_function_io.F90 new file mode 100644 index 000000000..907a8dc82 --- /dev/null +++ b/src/specfem3D/green_function_io.F90 @@ -0,0 +1,542 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---- +!---- Green function database: HDF5 file creation, buffered writing, +!---- and time loop integration. +!---- +!---- Each MPI rank independently creates/writes one HDF5 file per +!---- tagged element: {GF_DATABASE_PATH}/elements/{morton_hex}/{net}.{sta}.h5 +!---- +!---- Dataset "displacement": shape (3, 3, 5, 5, 5, nt_sub) in Fortran order +!---- where nt_sub = NSTEP / GF_SUBSAMPLE_STEP +!---- dim 1: force component (N, E, Z) — GF_NCOMP_FORCE +!---- dim 2: displacement component (x, y, z) — GF_NCOMP_DISP +!---- dims 3-5: GLL indices (NGLLX, NGLLY, NGLLZ) +!---- dim 6: subsampled time snapshots +!---- +!---- Each simulation run writes into one force component slice. +!---- Files are created on the first run and opened (not truncated) +!---- on subsequent runs, preserving previously written components. +!---- +!---- Chunking: (1, 3, 5, 5, 5, min(GF_BUFFER_SIZE, nt_sub)) +!---- one force component per chunk, aligned to the write pattern +!---- + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_init_hdf5() + +! Initializes HDF5 files and allocates the write buffer. +! Called from prepare_green_function_storage() during solver setup. +! +! On the first run (N component), creates new HDF5 files with fill value 0. +! On subsequent runs (E, Z), opens existing files without truncating. + +#ifdef USE_HDF5 + use hdf5 + + use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IMAIN,MAX_STRING_LEN, & + GF_NCOMP_DISP,GF_NCOMP_FORCE + + use specfem_par, only: NSTEP,DT,myrank + + use shared_parameters, only: GF_DATABASE_PATH,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE + + use green_function_par, only: & + gf_nelem_local, gf_morton_hex, gf_center_xyz, & + gf_buffer, gf_isnap, gf_ibuf, gf_nt_sub, & + gf_network_name, gf_station_name, & + gf_force_component, gf_f_cutoff, gf_hdur + + implicit none + + ! local variables + integer :: i, ier + integer :: hdferr + character(len=MAX_STRING_LEN) :: filepath + logical :: file_exists + + ! HDF5 identifiers (local — each file is opened and closed immediately) + integer(HID_T) :: fid, dspace_id, dset_id, dcpl_id + integer(HID_T) :: aspace_id, attr_id + + ! dataset dimensions (6D) + integer(HSIZE_T), dimension(6) :: dims, chunk_dims + integer(HSIZE_T), dimension(1) :: adim + + ! attribute values + double precision :: attr_dp(1) + integer :: attr_int(1) + + ! fill value + real :: fill_val_r + double precision :: fill_val_d + + ! nothing to do if no local elements + if (gf_nelem_local == 0) return + + ! compute number of subsampled timesteps + gf_nt_sub = NSTEP / GF_SUBSAMPLE_STEP + + ! initialize counters + gf_isnap = 0 + gf_ibuf = 0 + + ! allocate write buffer: (3, 5, 5, 5, bufsize, nelem) + ! element index is LAST so each element's time slices are contiguous + allocate(gf_buffer(GF_NCOMP_DISP, NGLLX, NGLLY, NGLLZ, GF_BUFFER_SIZE, gf_nelem_local), & + stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_buffer') + gf_buffer(:,:,:,:,:,:) = 0.0_CUSTOM_REAL + + ! initialize HDF5 Fortran interface + call h5open_f(hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error initializing HDF5') + + ! dataset dimensions: (GF_NCOMP_FORCE, GF_NCOMP_DISP, NGLLX, NGLLY, NGLLZ, nt_sub) + dims(1) = GF_NCOMP_FORCE + dims(2) = GF_NCOMP_DISP + dims(3) = NGLLX + dims(4) = NGLLY + dims(5) = NGLLZ + dims(6) = gf_nt_sub + + ! chunk dimensions: (1, 3, 5, 5, 5, min(buffer, nt_sub)) + ! one force component per chunk + chunk_dims(1) = 1 + chunk_dims(2) = GF_NCOMP_DISP + chunk_dims(3) = NGLLX + chunk_dims(4) = NGLLY + chunk_dims(5) = NGLLZ + chunk_dims(6) = min(GF_BUFFER_SIZE, gf_nt_sub) + + ! create or open one HDF5 file per local element + do i = 1, gf_nelem_local + filepath = trim(GF_DATABASE_PATH) // '/elements/' // & + gf_morton_hex(i) // '/' // & + trim(gf_network_name) // '.' // trim(gf_station_name) // '.h5' + + ! check if file already exists (from a previous force component run) + inquire(file=trim(filepath), exist=file_exists) + + if (file_exists) then + ! open existing file — preserves data from other force components + call h5fopen_f(trim(filepath), H5F_ACC_RDWR_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error opening existing GF HDF5 file') + ! close immediately — we just verified it opens correctly + call h5fclose_f(fid, hdferr) + else + ! create new file with dataset + call h5fcreate_f(trim(filepath), H5F_ACC_TRUNC_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating GF HDF5 file') + + ! create dataspace (6D) + call h5screate_simple_f(6, dims, dspace_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating GF HDF5 dataspace') + + ! create dataset creation property list with chunking and fill value + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating GF HDF5 property list') + + call h5pset_chunk_f(dcpl_id, 6, chunk_dims, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error setting GF HDF5 chunk size') + + ! set fill value to 0 so unwritten force components are zero + if (CUSTOM_REAL == 4) then + fill_val_r = 0.0 + call h5pset_fill_value_f(dcpl_id, H5T_NATIVE_REAL, fill_val_r, hdferr) + else + fill_val_d = 0.0d0 + call h5pset_fill_value_f(dcpl_id, H5T_NATIVE_DOUBLE, fill_val_d, hdferr) + endif + + ! create dataset + if (CUSTOM_REAL == 4) then + call h5dcreate_f(fid, 'displacement', H5T_NATIVE_REAL, dspace_id, dset_id, hdferr, & + dcpl_id=dcpl_id) + else + call h5dcreate_f(fid, 'displacement', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr, & + dcpl_id=dcpl_id) + endif + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating GF HDF5 dataset') + + ! write attributes on the dataset + adim(1) = 1 + + ! dt (double) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'dt', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = DT + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! subsample_step (integer) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'subsample_step', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = GF_SUBSAMPLE_STEP + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! nsnap (integer) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'nsnap', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = gf_nt_sub + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! centroid coordinates cx, cy, cz (double) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'cx', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(1, i)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'cy', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(2, i)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'cz', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(3, i)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! hdur (double) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'hdur', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_hdur + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! f_cutoff (double) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'f_cutoff', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_f_cutoff + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! ngll (integer) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(dset_id, 'ngll', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NGLLX + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! close dataset, property list, dataspace, file + call h5dclose_f(dset_id, hdferr) + call h5pclose_f(dcpl_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + call h5fclose_f(fid, hdferr) + endif + enddo + + ! user output + if (myrank == 0) then + write(IMAIN,*) + write(IMAIN,*) 'Green function database: HDF5 files initialized' + write(IMAIN,*) ' force component: ', gf_force_component + write(IMAIN,*) ' subsampled timesteps: ', gf_nt_sub + write(IMAIN,*) ' buffer size: ', GF_BUFFER_SIZE + write(IMAIN,*) ' chunk time dimension: ', min(GF_BUFFER_SIZE, gf_nt_sub) + write(IMAIN,*) ' dataset shape: (', GF_NCOMP_FORCE, ',', GF_NCOMP_DISP, ',', & + NGLLX, ',', NGLLY, ',', NGLLZ, ',', gf_nt_sub, ')' + write(IMAIN,*) + call flush_IMAIN() + endif + +#else + ! no HDF5 support compiled + use shared_parameters, only: GF_DATABASE_ENABLED + use specfem_par, only: myrank + implicit none + if (GF_DATABASE_ENABLED) then + call exit_MPI(myrank, & + 'GF_DATABASE_ENABLED requires HDF5. Recompile with --with-hdf5') + endif +#endif + + end subroutine gf_init_hdf5 + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_write_timestep(it) + +! Called from iterate_time() at every timestep. +! Checks if this timestep should be subsampled, extracts displacement +! into the buffer, and flushes when the buffer is full. + +#ifdef USE_HDF5 + + use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,GF_NCOMP_DISP + + use shared_parameters, only: GF_DATABASE_ENABLED,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE + + use specfem_par_crustmantle, only: & + ibool => ibool_crust_mantle, & + displ => displ_crust_mantle + + use green_function_par, only: & + gf_nelem_local, gf_local_elements, & + gf_buffer, gf_isnap, gf_ibuf + + implicit none + + integer, intent(in) :: it + + ! local variables + integer :: ielem, ispec, i, j, k, iglob + + ! skip if no local elements or GF not enabled + if (.not. GF_DATABASE_ENABLED) return + if (gf_nelem_local == 0) return + + ! check if this timestep is a subsampled output step + if (mod(it, GF_SUBSAMPLE_STEP) /= 0) return + + ! increment buffer position + gf_ibuf = gf_ibuf + 1 + gf_isnap = gf_isnap + 1 + + ! extract displacement for all tagged elements into buffer + do ielem = 1, gf_nelem_local + ispec = gf_local_elements(ielem) + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool(i, j, k, ispec) + gf_buffer(1, i, j, k, gf_ibuf, ielem) = displ(1, iglob) + gf_buffer(2, i, j, k, gf_ibuf, ielem) = displ(2, iglob) + gf_buffer(3, i, j, k, gf_ibuf, ielem) = displ(3, iglob) + enddo + enddo + enddo + enddo + + ! flush buffer if full + if (gf_ibuf == GF_BUFFER_SIZE) then + call gf_flush_buffer() + endif + +#else + implicit none + integer, intent(in) :: it + ! unused + integer :: idummy + idummy = it +#endif + + end subroutine gf_write_timestep + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_flush_buffer() + +! Writes the current buffer contents to HDF5 files via hyperslab writes. +! Each element's data is written to its own file. +! +! The buffer contains displacement for ONE force component (the current run). +! The hyperslab offset uses gf_force_component to write into the correct +! slice of the 6D dataset. + +#ifdef USE_HDF5 + use hdf5 + use iso_c_binding, only: c_loc + + use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,MAX_STRING_LEN, & + GF_NCOMP_DISP + + use specfem_par, only: myrank + + use shared_parameters, only: GF_DATABASE_PATH + + use green_function_par, only: & + gf_nelem_local, gf_morton_hex, & + gf_buffer, gf_isnap, gf_ibuf, & + gf_network_name, gf_station_name, & + gf_force_component + + implicit none + + ! local variables + integer :: ielem, hdferr + character(len=MAX_STRING_LEN) :: filepath + + ! HDF5 identifiers + integer(HID_T) :: fid, dset_id, fspace_id, mspace_id + + ! hyperslab selection (6D file dataset) + integer(HSIZE_T), dimension(6) :: hs_offset, hs_count + + ! memory dataspace dimensions + integer(HSIZE_T), dimension(6) :: mem_dims + + ! nothing to flush + if (gf_ibuf == 0) return + if (gf_nelem_local == 0) return + + ! memory dimensions matching the hyperslab count + mem_dims(1) = 1 ! single force component + mem_dims(2) = GF_NCOMP_DISP + mem_dims(3) = NGLLX + mem_dims(4) = NGLLY + mem_dims(5) = NGLLZ + mem_dims(6) = gf_ibuf + + ! hyperslab count in file dataset (6D) + hs_count(1) = 1 ! one force component + hs_count(2) = GF_NCOMP_DISP + hs_count(3) = NGLLX + hs_count(4) = NGLLY + hs_count(5) = NGLLZ + hs_count(6) = gf_ibuf + + ! hyperslab offset (0-based) + hs_offset(1) = gf_force_component - 1 ! which force component (N=0, E=1, Z=2) + hs_offset(2) = 0 + hs_offset(3) = 0 + hs_offset(4) = 0 + hs_offset(5) = 0 + hs_offset(6) = gf_isnap - gf_ibuf ! time offset + + do ielem = 1, gf_nelem_local + filepath = trim(GF_DATABASE_PATH) // '/elements/' // & + gf_morton_hex(ielem) // '/' // & + trim(gf_network_name) // '.' // trim(gf_station_name) // '.h5' + + ! open existing file + call h5fopen_f(trim(filepath), H5F_ACC_RDWR_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error opening GF HDF5 file for writing') + + ! open dataset + call h5dopen_f(fid, 'displacement', dset_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error opening GF displacement dataset') + + ! get file dataspace (6D) and select hyperslab + call h5dget_space_f(dset_id, fspace_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error getting GF file dataspace') + + call h5sselect_hyperslab_f(fspace_id, H5S_SELECT_SET_F, hs_offset, hs_count, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error selecting GF HDF5 hyperslab') + + ! create memory dataspace matching hyperslab count + call h5screate_simple_f(6, mem_dims, mspace_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating GF memory dataspace') + + ! write data using c_loc to bypass Fortran rank checking + ! gf_buffer(:,:,:,:, 1:ibuf, ielem) is contiguous in memory + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dset_id, H5T_NATIVE_REAL, & + c_loc(gf_buffer(1, 1, 1, 1, 1, ielem)), hdferr, & + mem_space_id=mspace_id, file_space_id=fspace_id) + else + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, & + c_loc(gf_buffer(1, 1, 1, 1, 1, ielem)), hdferr, & + mem_space_id=mspace_id, file_space_id=fspace_id) + endif + if (hdferr /= 0) call exit_MPI(myrank, 'Error writing GF HDF5 data') + + ! close all + call h5sclose_f(mspace_id, hdferr) + call h5sclose_f(fspace_id, hdferr) + call h5dclose_f(dset_id, hdferr) + call h5fclose_f(fid, hdferr) + enddo + + ! reset buffer position + gf_ibuf = 0 + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_flush_buffer + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_finalize_hdf5() + +! Flushes any remaining buffer contents and cleans up. +! Called from finalize_simulation(). + +#ifdef USE_HDF5 + use constants, only: IMAIN + use specfem_par, only: myrank + + use shared_parameters, only: GF_DATABASE_ENABLED + + use green_function_par, only: & + gf_nelem_local, gf_buffer, gf_ibuf, gf_isnap, gf_nt_sub + + implicit none + + if (.not. GF_DATABASE_ENABLED) return + if (gf_nelem_local == 0) return + + ! flush remaining data in buffer + if (gf_ibuf > 0) then + call gf_flush_buffer() + endif + + ! verify all snapshots were written + if (myrank == 0) then + write(IMAIN,*) + write(IMAIN,*) 'Green function database: HDF5 finalized' + write(IMAIN,*) ' total snapshots written: ', gf_isnap, ' / ', gf_nt_sub + write(IMAIN,*) + call flush_IMAIN() + endif + + ! deallocate buffer + if (allocated(gf_buffer)) deallocate(gf_buffer) + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_finalize_hdf5 diff --git a/src/specfem3D/green_function_par.F90 b/src/specfem3D/green_function_par.F90 index 88b6ae6e7..75109fbef 100644 --- a/src/specfem3D/green_function_par.F90 +++ b/src/specfem3D/green_function_par.F90 @@ -56,4 +56,13 @@ module green_function_par character(len=16), dimension(:), allocatable :: gf_morton_hex ! hex string per local element real(kind=CUSTOM_REAL), dimension(:,:), allocatable :: gf_center_xyz ! (3, gf_nelem_local) center coords + ! HDF5 I/O (Stage 7) + ! write buffer: (3, NGLLX, NGLLY, NGLLZ, GF_BUFFER_SIZE, gf_nelem_local) + ! element index is LAST so each element's time slices are contiguous in memory + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:,:), allocatable, target :: gf_buffer + + integer :: gf_isnap = 0 ! total subsampled snapshots written so far + integer :: gf_ibuf = 0 ! current position in buffer (1..GF_BUFFER_SIZE) + integer :: gf_nt_sub = 0 ! total number of subsampled timesteps + end module green_function_par diff --git a/src/specfem3D/iterate_time.F90 b/src/specfem3D/iterate_time.F90 index 48f4d7f99..24d514de8 100644 --- a/src/specfem3D/iterate_time.F90 +++ b/src/specfem3D/iterate_time.F90 @@ -218,6 +218,9 @@ subroutine iterate_time() ! write the seismograms with time shift (GPU_MODE transfer included) call write_seismograms() + ! Green function database: buffer displacement at subsampled timesteps + call gf_write_timestep(it) + ! adjoint simulations: kernels ! attention: for GPU_MODE and ANISOTROPIC_KL it is necessary to use resort_array (see lines 265-268) if (SIMULATION_TYPE == 3) then diff --git a/src/specfem3D/prepare_green_function_storage.f90 b/src/specfem3D/prepare_green_function_storage.f90 index 56784bc90..ad8e7e930 100644 --- a/src/specfem3D/prepare_green_function_storage.f90 +++ b/src/specfem3D/prepare_green_function_storage.f90 @@ -48,4 +48,7 @@ subroutine prepare_green_function_storage() ! write centroids.bin and manifest.csv call gf_write_manifest() + ! initialize HDF5 files and allocate write buffer + call gf_init_hdf5() + end subroutine prepare_green_function_storage diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index a3abef813..7277e78a2 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -92,6 +92,7 @@ specfem3D_SOLVER_OBJECTS += \ $O/green_function_locate.solverstatic.o \ $O/green_function_expand.solverstatic.o \ $O/green_function_morton.solverstatic.o \ + $O/green_function_io.solverstatic.o \ $O/prepare_green_function_storage.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ @@ -470,6 +471,7 @@ $O/green_function_stf.solverstatic.o: $O/green_function_par.solverstatic_module. $O/green_function_locate.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_expand.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_morton.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_io.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here From dd51f65eef4aa1ea62c20bb42883f5caa2496bbb Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sun, 10 May 2026 01:28:01 -0400 Subject: [PATCH 06/20] Stage 8: metadata, completion tracking, and rotation reversal Add per-element coordinates.h5 (GLL node coords), shared mesh_info.h5 (simulation params, topography, ellipticity), and per-station metadata (lat/lon/depth, STF). Track completion via computed_N/E/Z/ALL attributes on element station files to enable skip-on-rerun (GF_OVERWRITE parameter). Reverse rotation for reciprocity, multiply displacement by scale_displ for float32 precision, and fix station name parsing from FORCESOLUTION. Add Python post-processing: gf_build_manifest.py (centroids + completion summary) and read_centroids.py (binary reader example). --- src/shared/broadcast_computed_parameters.f90 | 6 +- src/shared/read_parameter_file.F90 | 1 + src/shared/shared_par.f90 | 1 + src/specfem3D/green_function_detect.F90 | 55 +- src/specfem3D/green_function_io.F90 | 91 ++- src/specfem3D/green_function_metadata.F90 | 731 ++++++++++++++++++ src/specfem3D/green_function_par.F90 | 6 + .../prepare_green_function_storage.f90 | 10 +- src/specfem3D/prepare_timerun.F90 | 5 + src/specfem3D/rules.mk | 2 + utils/green_function/gf_build_manifest.py | 171 ++++ utils/green_function/read_centroids.py | 52 ++ 12 files changed, 1107 insertions(+), 24 deletions(-) create mode 100644 src/specfem3D/green_function_metadata.F90 create mode 100755 utils/green_function/gf_build_manifest.py create mode 100755 utils/green_function/read_centroids.py diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90 index 4956d309d..a7cb19150 100644 --- a/src/shared/broadcast_computed_parameters.f90 +++ b/src/shared/broadcast_computed_parameters.f90 @@ -37,7 +37,7 @@ subroutine broadcast_computed_parameters() integer, parameter :: nparam_i = 54 integer, dimension(nparam_i) :: bcast_integer - integer, parameter :: nparam_l = 82 + integer, parameter :: nparam_l = 83 logical, dimension(nparam_l) :: bcast_logical integer, parameter :: nparam_dp = 42 @@ -117,7 +117,8 @@ subroutine broadcast_computed_parameters() FULL_GRAVITY, USE_SINSQ_STF, & HDF5_ENABLED, HDF5_FOR_MOVIES, OUTPUT_SEISMOS_HDF5, & ATTENUATION_3D_BERKELEY, & - GF_DATABASE_ENABLED /) + GF_DATABASE_ENABLED, & + GF_OVERWRITE /) bcast_double_precision = (/ & DT, & @@ -381,6 +382,7 @@ subroutine broadcast_computed_parameters() OUTPUT_SEISMOS_HDF5 = bcast_logical(80) ATTENUATION_3D_BERKELEY = bcast_logical(81) GF_DATABASE_ENABLED = bcast_logical(82) + GF_OVERWRITE = bcast_logical(83) ! double precisions DT = bcast_double_precision(1) diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index fb4dd5e8b..077d9af23 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -401,6 +401,7 @@ subroutine read_parameter_file() call read_value_integer(GF_SUBSAMPLE_STEP, 'GF_SUBSAMPLE_STEP', ier); ier = 0 call read_value_integer(GF_BUFFER_SIZE, 'GF_BUFFER_SIZE', ier); ier = 0 call read_value_integer(GF_NEIGHBOR_SHELLS, 'GF_NEIGHBOR_SHELLS', ier); ier = 0 + call read_value_logical(GF_OVERWRITE, 'GF_OVERWRITE', ier); ier = 0 endif ! closes parameter file diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index 8d8bbb024..8f9968203 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -237,6 +237,7 @@ module shared_input_parameters integer :: GF_SUBSAMPLE_STEP = 4 integer :: GF_BUFFER_SIZE = 100 integer :: GF_NEIGHBOR_SHELLS = 1 + logical :: GF_OVERWRITE = .false. end module shared_input_parameters diff --git a/src/specfem3D/green_function_detect.F90 b/src/specfem3D/green_function_detect.F90 index 3d552b659..1b51f6074 100644 --- a/src/specfem3D/green_function_detect.F90 +++ b/src/specfem3D/green_function_detect.F90 @@ -41,13 +41,14 @@ subroutine gf_detect_force_component() use specfem_par, only: myrank, & comp_dir_vect_source_E,comp_dir_vect_source_N,comp_dir_vect_source_Z_UP - use green_function_par, only: gf_force_component,gf_network_name,gf_station_name + use green_function_par, only: gf_force_component,gf_network_name,gf_station_name, & + gf_station_lat,gf_station_lon,gf_station_depth implicit none ! local variables - character(len=MAX_STRING_LEN) :: string,FORCESOLUTION,path_to_add - integer :: ier,idot + character(len=MAX_STRING_LEN) :: string,header_line,FORCESOLUTION,path_to_add + integer :: ier,idot,ipos logical :: has_N,has_E,has_Z integer :: ncomp @@ -93,17 +94,50 @@ subroutine gf_detect_force_component() endif ! read header line (e.g., "FORCE IU.SJG") - read(IIN,"(a)") string + read(IIN,"(a)") header_line ! skip empty lines - do while (len_trim(string) == 0) - read(IIN,"(a)") string + do while (len_trim(header_line) == 0) + read(IIN,"(a)") header_line enddo + + ! skip time shift line + read(IIN,"(a)") string + ! skip f0/hdur line + read(IIN,"(a)") string + + ! read latitude + read(IIN,"(a)") string + ipos = index(string,':') + if (ipos > 1 .and. ipos < len_trim(string)) then + read(string(ipos+1:len_trim(string)),*) gf_station_lat + else + read(string(10:len_trim(string)),*) gf_station_lat + endif + + ! read longitude + read(IIN,"(a)") string + ipos = index(string,':') + if (ipos > 1 .and. ipos < len_trim(string)) then + read(string(ipos+1:len_trim(string)),*) gf_station_lon + else + read(string(11:len_trim(string)),*) gf_station_lon + endif + + ! read depth + read(IIN,"(a)") string + ipos = index(string,':') + if (ipos > 1 .and. ipos < len_trim(string)) then + read(string(ipos+1:len_trim(string)),*) gf_station_depth + else + read(string(7:len_trim(string)),*) gf_station_depth + endif + close(IIN) - ! extract station identity: everything after "FORCE" prefix, trimmed + ! extract station identity from the saved header line ! header format: "FORCE IU.SJG" or "FORCE 001" ! skip the first 5 characters ("FORCE"), then trim leading spaces - string = adjustl(string(6:)) + string = adjustl(header_line(6:)) ! split on '.' to get network and station idot = index(trim(string), '.') @@ -132,8 +166,11 @@ subroutine gf_detect_force_component() call flush_IMAIN() endif - ! broadcast station identity to all ranks + ! broadcast station identity and location to all ranks call bcast_all_ch(gf_network_name, 8) call bcast_all_ch(gf_station_name, 32) + call bcast_all_dp(gf_station_lat, 1) + call bcast_all_dp(gf_station_lon, 1) + call bcast_all_dp(gf_station_depth, 1) end subroutine gf_detect_force_component diff --git a/src/specfem3D/green_function_io.F90 b/src/specfem3D/green_function_io.F90 index 907a8dc82..9faeca1c7 100644 --- a/src/specfem3D/green_function_io.F90 +++ b/src/specfem3D/green_function_io.F90 @@ -67,13 +67,14 @@ subroutine gf_init_hdf5() use specfem_par, only: NSTEP,DT,myrank - use shared_parameters, only: GF_DATABASE_PATH,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE + use shared_parameters, only: GF_DATABASE_PATH,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE,GF_OVERWRITE use green_function_par, only: & gf_nelem_local, gf_morton_hex, gf_center_xyz, & gf_buffer, gf_isnap, gf_ibuf, gf_nt_sub, & gf_network_name, gf_station_name, & - gf_force_component, gf_f_cutoff, gf_hdur + gf_force_component, gf_f_cutoff, gf_hdur, & + gf_elem_active implicit none @@ -82,6 +83,7 @@ subroutine gf_init_hdf5() integer :: hdferr character(len=MAX_STRING_LEN) :: filepath logical :: file_exists + integer :: nskipped ! HDF5 identifiers (local — each file is opened and closed immediately) integer(HID_T) :: fid, dspace_id, dset_id, dcpl_id @@ -116,6 +118,11 @@ subroutine gf_init_hdf5() if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_buffer') gf_buffer(:,:,:,:,:,:) = 0.0_CUSTOM_REAL + ! allocate element active mask (completion tracking) + allocate(gf_elem_active(gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_elem_active') + gf_elem_active(:) = .true. + ! initialize HDF5 Fortran interface call h5open_f(hdferr) if (hdferr /= 0) call exit_MPI(myrank, 'Error initializing HDF5') @@ -150,7 +157,19 @@ subroutine gf_init_hdf5() ! open existing file — preserves data from other force components call h5fopen_f(trim(filepath), H5F_ACC_RDWR_F, fid, hdferr) if (hdferr /= 0) call exit_MPI(myrank, 'Error opening existing GF HDF5 file') - ! close immediately — we just verified it opens correctly + + ! check completion: if computed_ALL == 1 and not overwriting, skip this element + if (.not. GF_OVERWRITE) then + call h5aopen_f(fid, 'computed_ALL', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + if (attr_int(1) == 1) then + gf_elem_active(i) = .false. + endif + endif + endif + call h5fclose_f(fid, hdferr) else ! create new file with dataset @@ -260,14 +279,48 @@ subroutine gf_init_hdf5() call h5aclose_f(attr_id, hdferr) call h5sclose_f(aspace_id, hdferr) - ! close dataset, property list, dataspace, file + ! close dataset, property list, dataspace call h5dclose_f(dset_id, hdferr) call h5pclose_f(dcpl_id, hdferr) call h5sclose_f(dspace_id, hdferr) + + ! completion tracking attributes (on the file root, all zero initially) + attr_int(1) = 0 + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'computed_N', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'computed_E', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'computed_Z', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'computed_ALL', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + call h5fclose_f(fid, hdferr) endif enddo + ! count skipped elements + nskipped = 0 + do i = 1, gf_nelem_local + if (.not. gf_elem_active(i)) nskipped = nskipped + 1 + enddo + ! user output if (myrank == 0) then write(IMAIN,*) @@ -278,6 +331,9 @@ subroutine gf_init_hdf5() write(IMAIN,*) ' chunk time dimension: ', min(GF_BUFFER_SIZE, gf_nt_sub) write(IMAIN,*) ' dataset shape: (', GF_NCOMP_FORCE, ',', GF_NCOMP_DISP, ',', & NGLLX, ',', NGLLY, ',', NGLLZ, ',', gf_nt_sub, ')' + if (nskipped > 0) then + write(IMAIN,*) ' elements skipped (already completed): ', nskipped + endif write(IMAIN,*) call flush_IMAIN() endif @@ -311,13 +367,15 @@ subroutine gf_write_timestep(it) use shared_parameters, only: GF_DATABASE_ENABLED,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE + use specfem_par, only: scale_displ + use specfem_par_crustmantle, only: & ibool => ibool_crust_mantle, & displ => displ_crust_mantle use green_function_par, only: & gf_nelem_local, gf_local_elements, & - gf_buffer, gf_isnap, gf_ibuf + gf_buffer, gf_isnap, gf_ibuf, gf_elem_active implicit none @@ -325,6 +383,7 @@ subroutine gf_write_timestep(it) ! local variables integer :: ielem, ispec, i, j, k, iglob + real(kind=CUSTOM_REAL) :: scale ! skip if no local elements or GF not enabled if (.not. GF_DATABASE_ENABLED) return @@ -337,16 +396,20 @@ subroutine gf_write_timestep(it) gf_ibuf = gf_ibuf + 1 gf_isnap = gf_isnap + 1 + ! scale_displ converts non-dimensional displacement to meters + scale = real(scale_displ, kind=CUSTOM_REAL) + ! extract displacement for all tagged elements into buffer do ielem = 1, gf_nelem_local + if (.not. gf_elem_active(ielem)) cycle ispec = gf_local_elements(ielem) do k = 1, NGLLZ do j = 1, NGLLY do i = 1, NGLLX iglob = ibool(i, j, k, ispec) - gf_buffer(1, i, j, k, gf_ibuf, ielem) = displ(1, iglob) - gf_buffer(2, i, j, k, gf_ibuf, ielem) = displ(2, iglob) - gf_buffer(3, i, j, k, gf_ibuf, ielem) = displ(3, iglob) + gf_buffer(1, i, j, k, gf_ibuf, ielem) = displ(1, iglob) * scale + gf_buffer(2, i, j, k, gf_ibuf, ielem) = displ(2, iglob) * scale + gf_buffer(3, i, j, k, gf_ibuf, ielem) = displ(3, iglob) * scale enddo enddo enddo @@ -395,7 +458,7 @@ subroutine gf_flush_buffer() gf_nelem_local, gf_morton_hex, & gf_buffer, gf_isnap, gf_ibuf, & gf_network_name, gf_station_name, & - gf_force_component + gf_force_component, gf_elem_active implicit none @@ -441,6 +504,8 @@ subroutine gf_flush_buffer() hs_offset(6) = gf_isnap - gf_ibuf ! time offset do ielem = 1, gf_nelem_local + if (.not. gf_elem_active(ielem)) cycle + filepath = trim(GF_DATABASE_PATH) // '/elements/' // & gf_morton_hex(ielem) // '/' // & trim(gf_network_name) // '.' // trim(gf_station_name) // '.h5' @@ -510,7 +575,7 @@ subroutine gf_finalize_hdf5() use shared_parameters, only: GF_DATABASE_ENABLED use green_function_par, only: & - gf_nelem_local, gf_buffer, gf_ibuf, gf_isnap, gf_nt_sub + gf_nelem_local, gf_buffer, gf_ibuf, gf_isnap, gf_nt_sub, gf_elem_active implicit none @@ -522,6 +587,9 @@ subroutine gf_finalize_hdf5() call gf_flush_buffer() endif + ! update completion flags on element and station files + call gf_update_completion_flags() + ! verify all snapshots were written if (myrank == 0) then write(IMAIN,*) @@ -531,8 +599,9 @@ subroutine gf_finalize_hdf5() call flush_IMAIN() endif - ! deallocate buffer + ! deallocate buffer and active mask if (allocated(gf_buffer)) deallocate(gf_buffer) + if (allocated(gf_elem_active)) deallocate(gf_elem_active) #else implicit none diff --git a/src/specfem3D/green_function_metadata.F90 b/src/specfem3D/green_function_metadata.F90 new file mode 100644 index 000000000..e8f8aad3e --- /dev/null +++ b/src/specfem3D/green_function_metadata.F90 @@ -0,0 +1,731 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!---- +!---- Green function database: metadata, coordinate, and mesh info writing. +!---- +!---- Per-element coordinates.h5: full GLL node coordinates (3,5,5,5) +!---- Per-database mesh_info.h5: topography + ellipticity arrays (write-once) +!---- Per-station stations/{net}.{sta}.h5: simulation params, STF, station info +!---- Completion tracking: computed_N/E/Z/ALL attributes on station displacement files +!---- + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_write_coordinates() + +! Writes per-element coordinate files: +! {GF_DATABASE_PATH}/elements/{morton_hex}/coordinates.h5 +! +! Contains the full GLL node coordinates (3, NGLLX, NGLLY, NGLLZ) +! needed for point-in-element location and Lagrange interpolation at extraction time. +! +! Idempotent: skips writing if the file already exists (another station run +! may have created it — coordinates are deterministic per element). + +#ifdef USE_HDF5 + use hdf5 + + use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,IMAIN,MAX_STRING_LEN + + use specfem_par, only: myrank + + use shared_parameters, only: GF_DATABASE_PATH + + use specfem_par_crustmantle, only: & + ibool => ibool_crust_mantle, & + xstore => xstore_crust_mantle, & + ystore => ystore_crust_mantle, & + zstore => zstore_crust_mantle + + use green_function_par, only: & + gf_nelem_local, gf_local_elements, gf_morton_hex, gf_center_xyz + + implicit none + + integer :: ielem, ispec, i, j, k, iglob, hdferr, ier + character(len=MAX_STRING_LEN) :: filepath + logical :: file_exists + + ! HDF5 identifiers + integer(HID_T) :: fid, dspace_id, dset_id, aspace_id, attr_id + integer(HSIZE_T), dimension(4) :: dims + integer(HSIZE_T), dimension(1) :: adim + + ! coordinate buffer for one element + real(kind=CUSTOM_REAL), dimension(3, NGLLX, NGLLY, NGLLZ), target :: xyz_elem + + ! attribute values + double precision :: attr_dp(1) + + integer :: nwritten + + if (gf_nelem_local == 0) return + + ! initialize HDF5 Fortran interface (reference-counted, safe to call multiple times) + call h5open_f(hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error initializing HDF5 in gf_write_coordinates') + + dims(1) = 3 + dims(2) = NGLLX + dims(3) = NGLLY + dims(4) = NGLLZ + + adim(1) = 1 + + nwritten = 0 + + do ielem = 1, gf_nelem_local + filepath = trim(GF_DATABASE_PATH) // '/elements/' // & + gf_morton_hex(ielem) // '/coordinates.h5' + + ! skip if file already exists (idempotent) + inquire(file=trim(filepath), exist=file_exists) + if (file_exists) cycle + + ispec = gf_local_elements(ielem) + + ! extract GLL node coordinates + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool(i, j, k, ispec) + xyz_elem(1, i, j, k) = xstore(iglob) + xyz_elem(2, i, j, k) = ystore(iglob) + xyz_elem(3, i, j, k) = zstore(iglob) + enddo + enddo + enddo + + ! create HDF5 file + call h5fcreate_f(trim(filepath), H5F_ACC_TRUNC_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating coordinates.h5') + + ! create dataset xyz (3, 5, 5, 5) + call h5screate_simple_f(4, dims, dspace_id, hdferr) + if (CUSTOM_REAL == 4) then + call h5dcreate_f(fid, 'xyz', H5T_NATIVE_REAL, dspace_id, dset_id, hdferr) + else + call h5dcreate_f(fid, 'xyz', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr) + endif + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating xyz dataset') + + ! write data + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dset_id, H5T_NATIVE_REAL, xyz_elem, dims, hdferr) + else + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, xyz_elem, dims, hdferr) + endif + if (hdferr /= 0) call exit_MPI(myrank, 'Error writing xyz dataset') + + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + + ! write centroid attributes for quick lookup + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'cx', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(1, ielem)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'cy', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(2, ielem)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'cz', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = dble(gf_center_xyz(3, ielem)) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! write morton_hex as attribute + call h5screate_f(H5S_SCALAR_F, aspace_id, hdferr) + call h5acreate_f(fid, 'morton_hex', H5T_NATIVE_CHARACTER, aspace_id, attr_id, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5fclose_f(fid, hdferr) + + nwritten = nwritten + 1 + enddo + + if (myrank == 0) then + write(IMAIN,*) 'Green function database: coordinate files written' + write(IMAIN,*) ' new files on rank 0: ', nwritten + write(IMAIN,*) + call flush_IMAIN() + endif + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_write_coordinates + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_write_mesh_info() + +! Writes the shared mesh info file: +! {GF_DATABASE_PATH}/mesh_info.h5 +! +! Contains topography (ibathy_topo) and ellipticity spline arrays needed +! for geographic-to-Cartesian coordinate conversion at extraction time. +! +! Written by rank 0 only. Idempotent: skips if file already exists. + +#ifdef USE_HDF5 + use hdf5 + + use constants, only: CUSTOM_REAL,NGLLX,IMAIN,MAX_STRING_LEN,NR_DENSITY + + use specfem_par, only: myrank, NSTEP, DT, & + ibathy_topo, NX_BATHY_VAL, NY_BATHY_VAL, & + nspl_ellip, rspl_ellip, ellipicity_spline, ellipicity_spline2, & + scale_displ + + use shared_parameters, only: GF_DATABASE_PATH, GF_SUBSAMPLE_STEP, GF_BUFFER_SIZE, & + GF_NEIGHBOR_SHELLS, & + TOPOGRAPHY, ELLIPTICITY, GRAVITY, ROTATION, ATTENUATION, & + RESOLUTION_TOPO_FILE, R_PLANET + + use green_function_par, only: gf_nt_sub + + implicit none + + character(len=MAX_STRING_LEN) :: filepath + logical :: file_exists + integer :: hdferr + + ! HDF5 identifiers + integer(HID_T) :: fid, dspace_id, dset_id, aspace_id, attr_id + integer(HSIZE_T), dimension(2) :: dims2 + integer(HSIZE_T), dimension(1) :: dims1, adim + + ! attribute values + double precision :: attr_dp(1) + integer :: attr_int(1) + + if (myrank /= 0) return + + ! initialize HDF5 Fortran interface (reference-counted, safe to call multiple times) + call h5open_f(hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error initializing HDF5 in gf_write_mesh_info') + + filepath = trim(GF_DATABASE_PATH) // '/mesh_info.h5' + + ! idempotent: skip if file already exists + inquire(file=trim(filepath), exist=file_exists) + if (file_exists) then + write(IMAIN,*) 'Green function database: mesh_info.h5 already exists, skipping' + write(IMAIN,*) + call flush_IMAIN() + return + endif + + ! create file + call h5fcreate_f(trim(filepath), H5F_ACC_TRUNC_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating mesh_info.h5') + + adim(1) = 1 + + ! write scale_displ + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'scale_displ', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = scale_displ + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! write R_PLANET + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'R_PLANET', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = R_PLANET + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! topography flag + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'TOPOGRAPHY', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + if (TOPOGRAPHY) then + attr_int(1) = 1 + else + attr_int(1) = 0 + endif + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! ellipticity flag + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'ELLIPTICITY', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + if (ELLIPTICITY) then + attr_int(1) = 1 + else + attr_int(1) = 0 + endif + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! simulation parameters + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'dt', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = DT + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'nstep', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NSTEP + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'nt_subsampled', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NSTEP / GF_SUBSAMPLE_STEP + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'subsample_step', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = GF_SUBSAMPLE_STEP + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'buffer_size', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = GF_BUFFER_SIZE + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'neighbor_shells', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = GF_NEIGHBOR_SHELLS + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'ngllx', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NGLLX + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! physics flags + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'rotation', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = merge(1, 0, ROTATION) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'attenuation', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = merge(1, 0, ATTENUATION) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'gravity', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = merge(1, 0, GRAVITY) + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! topography data + if (TOPOGRAPHY) then + ! NX_BATHY, NY_BATHY attributes + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'NX_BATHY', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NX_BATHY_VAL + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'NY_BATHY', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = NY_BATHY_VAL + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'RESOLUTION_TOPO_FILE', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = RESOLUTION_TOPO_FILE + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! ibathy_topo dataset (NX_BATHY, NY_BATHY) + dims2(1) = NX_BATHY_VAL + dims2(2) = NY_BATHY_VAL + call h5screate_simple_f(2, dims2, dspace_id, hdferr) + call h5dcreate_f(fid, 'ibathy_topo', H5T_NATIVE_INTEGER, dspace_id, dset_id, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating ibathy_topo dataset') + call h5dwrite_f(dset_id, H5T_NATIVE_INTEGER, ibathy_topo, dims2, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error writing ibathy_topo dataset') + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + endif + + ! ellipticity data + if (ELLIPTICITY) then + ! nspl attribute + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'nspl', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = nspl_ellip + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! rspl dataset + dims1(1) = nspl_ellip + call h5screate_simple_f(1, dims1, dspace_id, hdferr) + call h5dcreate_f(fid, 'rspl', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, rspl_ellip(1:nspl_ellip), dims1, hdferr) + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + + ! ellipicity_spline dataset + dims1(1) = nspl_ellip + call h5screate_simple_f(1, dims1, dspace_id, hdferr) + call h5dcreate_f(fid, 'ellipicity_spline', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, ellipicity_spline(1:nspl_ellip), dims1, hdferr) + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + + ! ellipicity_spline2 dataset + dims1(1) = nspl_ellip + call h5screate_simple_f(1, dims1, dspace_id, hdferr) + call h5dcreate_f(fid, 'ellipicity_spline2', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, ellipicity_spline2(1:nspl_ellip), dims1, hdferr) + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + endif + + call h5fclose_f(fid, hdferr) + + write(IMAIN,*) 'Green function database: mesh_info.h5 written' + if (TOPOGRAPHY) write(IMAIN,*) ' ibathy_topo: ', NX_BATHY_VAL, ' x ', NY_BATHY_VAL + if (ELLIPTICITY) write(IMAIN,*) ' ellipticity splines: nspl = ', nspl_ellip + write(IMAIN,*) ' scale_displ = ', scale_displ + write(IMAIN,*) + call flush_IMAIN() + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_write_mesh_info + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_write_station_metadata() + +! Writes per-station metadata file: +! {GF_DATABASE_PATH}/stations/{net}.{sta}.h5 +! +! Contains station-specific info, STF array, and source parameters. +! Simulation-wide parameters are in mesh_info.h5. +! Written by rank 0. Creates the file on first component run, +! skips if it already exists (idempotent — station info is the same across N/E/Z runs). + +#ifdef USE_HDF5 + use hdf5 + + use constants, only: CUSTOM_REAL,IMAIN,MAX_STRING_LEN + + use specfem_par, only: myrank, NSTEP, & + tshift_src, factor_force_source + + use shared_parameters, only: GF_DATABASE_PATH + + use green_function_par, only: & + gf_network_name, gf_station_name, & + gf_hdur, gf_f_cutoff, gf_stf, & + gf_station_lat, gf_station_lon, gf_station_depth + + implicit none + + character(len=MAX_STRING_LEN) :: filepath, dirpath + logical :: file_exists + integer :: hdferr + + ! HDF5 identifiers + integer(HID_T) :: fid, dspace_id, dset_id, aspace_id, attr_id + integer(HSIZE_T), dimension(1) :: dims1, adim + + ! attribute values + double precision :: attr_dp(1) + + if (myrank /= 0) return + + ! initialize HDF5 Fortran interface (reference-counted, safe to call multiple times) + call h5open_f(hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error initializing HDF5 in gf_write_station_metadata') + + ! create stations directory + dirpath = trim(GF_DATABASE_PATH) // '/stations' + call system('mkdir -p ' // trim(dirpath)) + + filepath = trim(dirpath) // '/' // & + trim(gf_network_name) // '.' // trim(gf_station_name) // '.h5' + + adim(1) = 1 + + ! idempotent: skip if file already exists (station info is identical across N/E/Z runs) + inquire(file=trim(filepath), exist=file_exists) + + if (.not. file_exists) then + ! create new file with station-specific metadata + call h5fcreate_f(trim(filepath), H5F_ACC_TRUNC_F, fid, hdferr) + if (hdferr /= 0) call exit_MPI(myrank, 'Error creating station metadata file') + + ! station info + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'latitude', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_station_lat + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'longitude', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_station_lon + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'depth', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_station_depth + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! STF parameters + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'hdur', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_hdur + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'f_cutoff', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = gf_f_cutoff + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'factor_force_source', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = factor_force_source(1) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'time_shift', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = tshift_src(1) + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! STF array + dims1(1) = NSTEP + call h5screate_simple_f(1, dims1, dspace_id, hdferr) + if (CUSTOM_REAL == 4) then + call h5dcreate_f(fid, 'stf', H5T_NATIVE_REAL, dspace_id, dset_id, hdferr) + call h5dwrite_f(dset_id, H5T_NATIVE_REAL, gf_stf, dims1, hdferr) + else + call h5dcreate_f(fid, 'stf', H5T_NATIVE_DOUBLE, dspace_id, dset_id, hdferr) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, gf_stf, dims1, hdferr) + endif + call h5dclose_f(dset_id, hdferr) + call h5sclose_f(dspace_id, hdferr) + + call h5fclose_f(fid, hdferr) + + write(IMAIN,*) 'Green function database: station metadata created' + write(IMAIN,*) ' file: ', trim(filepath) + write(IMAIN,*) + call flush_IMAIN() + else + write(IMAIN,*) 'Green function database: station metadata file exists, will update at finalization' + write(IMAIN,*) + call flush_IMAIN() + endif + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_write_station_metadata + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gf_update_completion_flags() + +! Updates completion flags on both the per-station metadata file and +! the per-element displacement files after successful data writing. +! +! Sets computed_N, computed_E, or computed_Z = 1 for the current force component. +! Called from gf_finalize_hdf5() after all data has been flushed. +! Rank 0 updates the station metadata file; each rank updates its own element files. + +#ifdef USE_HDF5 + use hdf5 + + use constants, only: IMAIN,MAX_STRING_LEN + + use specfem_par, only: myrank + + use shared_parameters, only: GF_DATABASE_PATH + + use green_function_par, only: & + gf_nelem_local, gf_morton_hex, & + gf_network_name, gf_station_name, gf_force_component + + implicit none + + integer :: ielem, hdferr + character(len=MAX_STRING_LEN) :: filepath + character(len=16) :: comp_attr_name + integer :: attr_int(1), comp_N, comp_E, comp_Z + integer(HSIZE_T), dimension(1) :: adim + + ! HDF5 identifiers + integer(HID_T) :: fid, attr_id, aspace_id + + adim(1) = 1 + + ! determine attribute name for current component + select case (gf_force_component) + case (1) + comp_attr_name = 'computed_N' + case (2) + comp_attr_name = 'computed_E' + case (3) + comp_attr_name = 'computed_Z' + end select + + ! update per-element station files + do ielem = 1, gf_nelem_local + filepath = trim(GF_DATABASE_PATH) // '/elements/' // & + gf_morton_hex(ielem) // '/' // & + trim(gf_network_name) // '.' // trim(gf_station_name) // '.h5' + + call h5fopen_f(trim(filepath), H5F_ACC_RDWR_F, fid, hdferr) + if (hdferr /= 0) cycle + + ! delete old attribute and write new value = 1 + call h5adelete_f(fid, trim(comp_attr_name), hdferr) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, trim(comp_attr_name), H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = 1 + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + + ! check if all components are done + comp_N = 0; comp_E = 0; comp_Z = 0 + + call h5aopen_f(fid, 'computed_N', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + comp_N = attr_int(1) + call h5aclose_f(attr_id, hdferr) + endif + + call h5aopen_f(fid, 'computed_E', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + comp_E = attr_int(1) + call h5aclose_f(attr_id, hdferr) + endif + + call h5aopen_f(fid, 'computed_Z', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + comp_Z = attr_int(1) + call h5aclose_f(attr_id, hdferr) + endif + + ! set computed_ALL if all three are done + if (comp_N == 1 .and. comp_E == 1 .and. comp_Z == 1) then + ! delete existing computed_ALL if present + call h5adelete_f(fid, 'computed_ALL', hdferr) + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 'computed_ALL', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) + attr_int(1) = 1 + call h5awrite_f(attr_id, H5T_NATIVE_INTEGER, attr_int, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + endif + + call h5fclose_f(fid, hdferr) + enddo + + if (myrank == 0) then + write(IMAIN,*) 'Green function database: completion flags updated' + write(IMAIN,*) ' component: ', trim(comp_attr_name), ' = 1' + write(IMAIN,*) + call flush_IMAIN() + endif + +#else + implicit none + ! stub — nothing to do without HDF5 +#endif + + end subroutine gf_update_completion_flags diff --git a/src/specfem3D/green_function_par.F90 b/src/specfem3D/green_function_par.F90 index 75109fbef..4bda3a2de 100644 --- a/src/specfem3D/green_function_par.F90 +++ b/src/specfem3D/green_function_par.F90 @@ -36,6 +36,9 @@ module green_function_par integer :: gf_force_component = 0 character(len=8) :: gf_network_name = '' character(len=32) :: gf_station_name = '' + double precision :: gf_station_lat = 0.d0 + double precision :: gf_station_lon = 0.d0 + double precision :: gf_station_depth = 0.d0 ! STF (Stage 3) ! precomputed source time function: Gaussian filtered with Butterworth lowpass @@ -56,6 +59,9 @@ module green_function_par character(len=16), dimension(:), allocatable :: gf_morton_hex ! hex string per local element real(kind=CUSTOM_REAL), dimension(:,:), allocatable :: gf_center_xyz ! (3, gf_nelem_local) center coords + ! Completion tracking (Stage 8) + logical, dimension(:), allocatable :: gf_elem_active ! .true. if element should be written + ! HDF5 I/O (Stage 7) ! write buffer: (3, NGLLX, NGLLY, NGLLZ, GF_BUFFER_SIZE, gf_nelem_local) ! element index is LAST so each element's time slices are contiguous in memory diff --git a/src/specfem3D/prepare_green_function_storage.f90 b/src/specfem3D/prepare_green_function_storage.f90 index ad8e7e930..81dfe69df 100644 --- a/src/specfem3D/prepare_green_function_storage.f90 +++ b/src/specfem3D/prepare_green_function_storage.f90 @@ -45,8 +45,14 @@ subroutine prepare_green_function_storage() ! create directory structure call gf_create_directories() - ! write centroids.bin and manifest.csv - call gf_write_manifest() + ! write per-element coordinate files (idempotent — skips if exists) + call gf_write_coordinates() + + ! write shared mesh info file (topography + ellipticity, rank 0, idempotent) + call gf_write_mesh_info() + + ! write per-station metadata file (rank 0, create-or-skip) + call gf_write_station_metadata() ! initialize HDF5 files and allocate write buffer call gf_init_hdf5() diff --git a/src/specfem3D/prepare_timerun.F90 b/src/specfem3D/prepare_timerun.F90 index 962623815..54f6764a6 100644 --- a/src/specfem3D/prepare_timerun.F90 +++ b/src/specfem3D/prepare_timerun.F90 @@ -641,6 +641,11 @@ subroutine prepare_timerun_constants() ! reconstructed wavefield together with +/- b_deltat will spin backward/forward b_two_omega_earth = real(2.d0 * TWO_PI / (HOURS_PER_DAY * SECONDS_PER_HOUR * scale_t_inv), kind=CUSTOM_REAL) endif + + ! Green function database: reciprocity requires reversed rotation + if (GF_DATABASE_ENABLED) then + two_omega_earth = -two_omega_earth + endif endif ! synchronizes processes diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index 7277e78a2..f565d22be 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -93,6 +93,7 @@ specfem3D_SOLVER_OBJECTS += \ $O/green_function_expand.solverstatic.o \ $O/green_function_morton.solverstatic.o \ $O/green_function_io.solverstatic.o \ + $O/green_function_metadata.solverstatic.o \ $O/prepare_green_function_storage.solverstatic.o \ $O/initialize_simulation.solverstatic.o \ $O/iterate_time.solverstatic.o \ @@ -472,6 +473,7 @@ $O/green_function_locate.solverstatic.o: $O/green_function_par.solverstatic_modu $O/green_function_expand.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_morton.solverstatic.o: $O/green_function_par.solverstatic_module.o $O/green_function_io.solverstatic.o: $O/green_function_par.solverstatic_module.o +$O/green_function_metadata.solverstatic.o: $O/green_function_par.solverstatic_module.o ### ### specfem3D - optimized flags and dependence on values from mesher here diff --git a/utils/green_function/gf_build_manifest.py b/utils/green_function/gf_build_manifest.py new file mode 100755 index 000000000..e51b395f1 --- /dev/null +++ b/utils/green_function/gf_build_manifest.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +""" +Build manifest and centroids files from a completed Green function database. + +Walks {GF_DATABASE_PATH}/elements/*/coordinates.h5 to extract centroids, +writes centroids.bin (binary) and manifest.csv (human-readable). +Also summarizes station completion status from {GF_DATABASE_PATH}/stations/*.h5. + +Usage: + python gf_build_manifest.py /path/to/gf_database/ + +Output files (written to GF_DATABASE_PATH): + centroids.bin — binary, 32 bytes/record: uint64 morton + 3x float64 (cx,cy,cz) + manifest.csv — text: morton_hex, cx, cy, cz + completion.txt — text summary of per-station completion status +""" + +import argparse +import struct +import sys +from pathlib import Path + +try: + import h5py + import numpy as np +except ImportError: + print("Error: h5py and numpy are required. Install with: pip install h5py numpy") + sys.exit(1) + + +def morton_hex_to_int(hex_str): + """Convert a 16-character hex string to a 64-bit integer.""" + return int(hex_str, 16) + + +def build_manifest(db_path): + """Build centroids.bin and manifest.csv from coordinates.h5 files.""" + elements_dir = db_path / "elements" + if not elements_dir.is_dir(): + print(f"Error: {elements_dir} does not exist") + return False + + # find all coordinates.h5 files + coord_files = sorted(elements_dir.glob("*/coordinates.h5")) + if not coord_files: + print(f"Warning: no coordinates.h5 files found in {elements_dir}") + return False + + print(f"Found {len(coord_files)} element coordinate files") + + records = [] + for coord_file in coord_files: + morton_hex = coord_file.parent.name + + with h5py.File(coord_file, "r") as f: + # read centroid from attributes + cx = float(np.asarray(f.attrs["cx"]).flat[0]) + cy = float(np.asarray(f.attrs["cy"]).flat[0]) + cz = float(np.asarray(f.attrs["cz"]).flat[0]) + + morton_int = morton_hex_to_int(morton_hex) + records.append((morton_int, morton_hex, cx, cy, cz)) + + # sort by Morton code + records.sort(key=lambda r: r[0]) + + # write centroids.bin + bin_path = db_path / "centroids.bin" + with open(bin_path, "wb") as f: + for morton_int, _, cx, cy, cz in records: + f.write(struct.pack(" list of (morton_hex, comp_N, comp_E, comp_Z, comp_ALL) + for sf in station_files: + station_name = sf.stem # e.g., "IU.SJG" + morton_hex = sf.parent.name + try: + with h5py.File(sf, "r") as f: + comp_n = int(np.asarray(f.attrs.get("computed_N", 0)).flat[0]) + comp_e = int(np.asarray(f.attrs.get("computed_E", 0)).flat[0]) + comp_z = int(np.asarray(f.attrs.get("computed_Z", 0)).flat[0]) + comp_all = int(np.asarray(f.attrs.get("computed_ALL", 0)).flat[0]) + except Exception as e: + print(f" {morton_hex}/{station_name} Error reading: {e}") + continue + + if station_name not in station_elements: + station_elements[station_name] = [] + station_elements[station_name].append((morton_hex, comp_n, comp_e, comp_z, comp_all)) + + # summarize per station + completion_lines = [] + print(f"\nElement completion status ({len(station_files)} element-station files, " + f"{len(station_elements)} stations):") + print(f"{'Station':<20s} {'Elements':>8s} {'Complete':>8s} {'N':>5s} {'E':>5s} {'Z':>5s}") + print("-" * 55) + + for station_name in sorted(station_elements): + elems = station_elements[station_name] + n_total = len(elems) + n_all = sum(1 for _, _, _, _, a in elems if a == 1) + n_n = sum(1 for _, n, _, _, _ in elems if n == 1) + n_e = sum(1 for _, _, e, _, _ in elems if e == 1) + n_z = sum(1 for _, _, _, z, _ in elems if z == 1) + + status_line = (f"{station_name:<20s} {n_total:>8d} {n_all:>8d} " + f"{n_n:>5d} {n_e:>5d} {n_z:>5d}") + print(f" {status_line}") + completion_lines.append(status_line) + + # write completion summary + completion_path = db_path / "completion.txt" + with open(completion_path, "w") as f: + f.write(f"Element completion status ({len(station_files)} element-station files, " + f"{len(station_elements)} stations)\n") + f.write(f"{'Station':<20s} {'Elements':>8s} {'Complete':>8s} {'N':>5s} {'E':>5s} {'Z':>5s}\n") + f.write("-" * 55 + "\n") + for line in completion_lines: + f.write(line + "\n") + print(f"Written {completion_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="Build manifest and centroids files from a Green function database" + ) + parser.add_argument( + "db_path", + type=Path, + help="Path to the Green function database directory", + ) + args = parser.parse_args() + + if not args.db_path.is_dir(): + print(f"Error: {args.db_path} is not a directory") + sys.exit(1) + + build_manifest(args.db_path) + check_element_completion(args.db_path) + + +if __name__ == "__main__": + main() diff --git a/utils/green_function/read_centroids.py b/utils/green_function/read_centroids.py new file mode 100755 index 000000000..eb5411ea8 --- /dev/null +++ b/utils/green_function/read_centroids.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +""" +Example: read centroids.bin and print the Morton codes + centroid coordinates. + +Usage: + python read_centroids.py /path/to/gf_database/centroids.bin +""" + +import argparse +import struct +import sys +from pathlib import Path + + +def read_centroids(filepath): + """Read centroids.bin and return list of (morton_hex, cx, cy, cz).""" + record_size = 32 # uint64 + 3x float64 + data = filepath.read_bytes() + + if len(data) % record_size != 0: + print(f"Warning: file size {len(data)} not a multiple of {record_size}") + + n = len(data) // record_size + records = [] + for i in range(n): + morton, cx, cy, cz = struct.unpack_from("20s} {'cy':>20s} {'cz':>20s}") + print("-" * 80) + for morton_hex, cx, cy, cz in records: + print(f"{morton_hex:<18s} {cx:>20.12e} {cy:>20.12e} {cz:>20.12e}") + + print(f"\n{len(records)} records") + + +if __name__ == "__main__": + main() From 5db4c1cb8c83cde5cbb54259705641dc89849df0 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sun, 10 May 2026 11:48:44 -0400 Subject: [PATCH 07/20] Updated the automatic setting of the halfduration --- src/specfem3D/green_function_stf.F90 | 11 ++++++----- src/specfem3D/setup_sources_receivers.f90 | 11 +++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/specfem3D/green_function_stf.F90 b/src/specfem3D/green_function_stf.F90 index 4b4449079..8ed595d49 100644 --- a/src/specfem3D/green_function_stf.F90 +++ b/src/specfem3D/green_function_stf.F90 @@ -44,7 +44,7 @@ subroutine gf_compute_stf() use constants, only: IMAIN,PI,CUSTOM_REAL,MAX_STRING_LEN use specfem_par, only: DT,NSTEP,t0,hdur,hdur_Gaussian,myrank,OUTPUT_FILES - use shared_parameters, only: GF_SUBSAMPLE_STEP + use shared_parameters, only: GF_SUBSAMPLE_STEP,T_min_period use green_function_par, only: gf_stf,gf_f_cutoff,gf_hdur implicit none @@ -62,10 +62,11 @@ subroutine gf_compute_stf() double precision, dimension(GF_FILTER_ORDER/2, 6) :: sos integer, parameter :: IOUT_STF = 71 - ! set half-duration: use the Gaussian decay half-duration (matching specfem convention) - ! hdur_Gaussian = hdur / SOURCE_DECAY_MIMIC_TRIANGLE - ! For GF, we use hdur_Gaussian(1) since NSOURCES==1 (validated in Stage 2) - gf_hdur = hdur_Gaussian(1) + ! set half-duration from mesh resolution: T_min_period / 10 + ! This keeps the Gaussian broadband while limiting unresolvable energy + ! (~14% above 1/T_min). The Butterworth lowpass handles anti-aliasing + ! for the subsampled output. + gf_hdur = T_min_period / 10.0d0 ! compute cutoff frequency: Nyquist of subsampled output gf_f_cutoff = 1.0d0 / (2.0d0 * DT * dble(GF_SUBSAMPLE_STEP)) diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index 3b5ff1f55..8590a80b3 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -740,7 +740,7 @@ subroutine setup_stf_constants() use specfem_par use specfem_par_movie - use shared_parameters, only: GF_DATABASE_ENABLED + use shared_parameters, only: GF_DATABASE_ENABLED,T_min_period implicit none ! local parameters @@ -844,12 +844,11 @@ subroutine setup_stf_constants() enddo endif - ! Green function database: ensure sufficient time buffer before source - ! The Butterworth lowpass filter introduces ringing that extends well - ! beyond the Gaussian's natural decay. We need a large buffer so the - ! filtered STF is essentially zero at the simulation start. + ! Green function database: ensure sufficient time buffer before source. + ! gf_hdur = T_min_period / 10, and the Gaussian decays to ~zero at ~3*gf_hdur. + ! The Butterworth filter broadens this slightly. Use ~5*gf_hdur = T_min/2. if (GF_DATABASE_ENABLED) then - t0 = max(t0, 15.0d0 * hdur(1)) + t0 = max(t0, T_min_period / 2.0d0) endif ! checks if user set USER_T0 to fix simulation start time From cd0ac48d15991e5a44c2ddf6395e4264f5126abb Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sun, 10 May 2026 14:23:04 -0400 Subject: [PATCH 08/20] Added validation script --- utils/green_function/gf_cross_validate.py | 689 ++++++++++++++++++++++ 1 file changed, 689 insertions(+) create mode 100644 utils/green_function/gf_cross_validate.py diff --git a/utils/green_function/gf_cross_validate.py b/utils/green_function/gf_cross_validate.py new file mode 100644 index 000000000..6f36de54a --- /dev/null +++ b/utils/green_function/gf_cross_validate.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +""" +Cross-validate Green function database against a forward simulation. + +Reads the GF displacement from the HDF5 database, interpolates to the +source location using Lagrange interpolation on GLL nodes, applies +reciprocity rotation, and compares against the forward seismogram. + +The forward simulation must use USE_FORCE_POINT_SOURCE = .true. with the +source placed at a location covered by the GF database (i.e., one of +the GF_LOCATIONS entries). + +Usage: + python gf_cross_validate.py [options] + +Example: + python gf_cross_validate.py \ + DB_TESTING/db_gen/OUTPUT_FILES/gf_database \ + DB_TESTING/forward/OUTPUT_FILES \ + --forward-solver-output DB_TESTING/forward/OUTPUT_FILES/output_solver.txt \ + --station IU.SJG +""" + +import argparse +import os +import re +import sys +from pathlib import Path +from math import sin, cos, radians, sqrt, pi + +import numpy as np + +try: + import h5py +except ImportError: + print("Error: h5py is required. Install with: pip install h5py") + sys.exit(1) + +try: + import matplotlib.pyplot as plt +except ImportError: + print("Error: matplotlib is required. Install with: pip install matplotlib") + sys.exit(1) + +try: + from obspy import read as obspy_read +except ImportError: + print("Error: obspy is required. Install with: pip install obspy") + sys.exit(1) + + +# GLL points for NGLL=5 (Gauss-Lobatto-Legendre quadrature nodes on [-1,1]) +GLL5 = np.array([-1.0, -0.6546536707079771, 0.0, 0.6546536707079771, 1.0]) + + +def lagrange_basis(xi, nodes): + """Compute Lagrange basis function values at point xi for given nodes. + + Returns array of length len(nodes) with L_i(xi) for each i. + """ + n = len(nodes) + L = np.ones(n) + for i in range(n): + for j in range(n): + if i != j: + L[i] *= (xi - nodes[j]) / (nodes[i] - nodes[j]) + return L + + +def interpolate_gll(data, xi, eta, gamma): + """Interpolate data on a 5x5x5 GLL element to point (xi, eta, gamma). + + Parameters + ---------- + data : ndarray, shape (nt, 5, 5, 5, 3, 3) + Displacement on GLL nodes. Dims are (time, k, j, i, disp, force) + where k,j,i are GLL node indices (reversed from Fortran i,j,k). + xi, eta, gamma : float + Reference coordinates in [-1, 1]. + + Returns + ------- + result : ndarray, shape (nt, 3, 3) + Interpolated values. + """ + Li = lagrange_basis(xi, GLL5) # basis for i-direction (dim 3) + Lj = lagrange_basis(eta, GLL5) # basis for j-direction (dim 2) + Lk = lagrange_basis(gamma, GLL5) # basis for k-direction (dim 1) + + # Contract over GLL dimensions (dims 1, 2, 3) + # data[t, k, j, i, d, f] -> sum over i,j,k with Li, Lj, Lk + result = np.einsum("tkjidf,i,j,k->tdf", data, Li, Lj, Lk) + return result + + +def parse_solver_output(filepath): + """Parse source info from specfem3D output_solver.txt. + + Returns dict with: xi, eta, gamma, x, y, z, nu1, nu2, nu3, lat, lon, depth + """ + text = Path(filepath).read_text() + + info = {} + + # Parse xi, eta, gamma + m = re.search( + r"at xi,eta,gamma coordinates\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", + text, + ) + if m: + info["xi"] = float(m.group(1)) + info["eta"] = float(m.group(2)) + info["gamma"] = float(m.group(3)) + + # Parse (x, y, z) + m = re.search( + r"at \(x,y,z\)\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text + ) + if m: + info["x"] = float(m.group(1)) + info["y"] = float(m.group(2)) + info["z"] = float(m.group(3)) + + # Parse nu1, nu2, nu3 (rotation matrix rows: North, East, Vertical) + for name in ["nu1", "nu2", "nu3"]: + m = re.search( + rf"{name}\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text + ) + if m: + info[name] = np.array( + [float(m.group(1)), float(m.group(2)), float(m.group(3))] + ) + + # Parse original source position + m = re.search(r"latitude:\s*([-\dE.+]+)", text) + if m: + info["lat"] = float(m.group(1)) + m = re.search(r"longitude:\s*([-\dE.+]+)", text) + if m: + info["lon"] = float(m.group(1)) + + # Parse force direction + m = re.search(r"East direction:\s*([-\dE.+]+)", text) + if m: + info["force_E"] = float(m.group(1)) + m = re.search(r"North direction:\s*([-\dE.+]+)", text) + if m: + info["force_N"] = float(m.group(1)) + m = re.search(r"Vertical direction:\s*([-\dE.+]+)", text) + if m: + info["force_Z"] = float(m.group(1)) + + return info + + +def lagrange_deriv(xi, nodes): + """Compute derivative of Lagrange basis functions at point xi.""" + n = len(nodes) + dL = np.zeros(n) + for i in range(n): + for j in range(n): + if j != i: + prod = 1.0 / (nodes[i] - nodes[j]) + for k in range(n): + if k != i and k != j: + prod *= (xi - nodes[k]) / (nodes[i] - nodes[k]) + dL[i] += prod + return dL + + +def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): + """Find reference coordinates (xi, eta, gamma) of a point in a spectral element. + + Uses Newton-Raphson iteration to invert the GLL coordinate mapping. + + Parameters + ---------- + xyz_elem : ndarray, shape (5, 5, 5, 3) + GLL node coordinates of the element. + target : ndarray, shape (3,) + Target point in Cartesian coordinates. + maxiter : int + Maximum Newton iterations. + tol : float + Convergence tolerance on residual norm. + + Returns + ------- + xi, eta, gamma : float + Reference coordinates. If all are in [-1, 1], the point is inside. + converged : bool + Whether Newton iteration converged. + """ + xi, eta, gamma = 0.0, 0.0, 0.0 + + for _ in range(maxiter): + Li = lagrange_basis(xi, GLL5) + Lj = lagrange_basis(eta, GLL5) + Lk = lagrange_basis(gamma, GLL5) + + # Current mapped position + current = np.array([ + np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, Lj, Lk) + for d in range(3) + ]) + + residual = target - current + if np.linalg.norm(residual) < tol: + return xi, eta, gamma, True + + # Jacobian via derivatives of Lagrange basis + dLi = lagrange_deriv(xi, GLL5) + dLj = lagrange_deriv(eta, GLL5) + dLk = lagrange_deriv(gamma, GLL5) + + J = np.zeros((3, 3)) + for d in range(3): + J[d, 0] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], dLi, Lj, Lk) + J[d, 1] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, dLj, Lk) + J[d, 2] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, Lj, dLk) + + dxi = np.linalg.solve(J, residual) + xi += dxi[0] + eta += dxi[1] + gamma += dxi[2] + + # Clamp to avoid divergence + xi = np.clip(xi, -1.5, 1.5) + eta = np.clip(eta, -1.5, 1.5) + gamma = np.clip(gamma, -1.5, 1.5) + + return xi, eta, gamma, False + + +def find_containing_element(gf_db_path, source_xyz): + """Find the element that contains the source point. + + Uses Newton-Raphson point location on candidate elements (sorted by + centroid distance). Returns the first element where the source maps to + reference coordinates within [-1, 1]. + + Returns (morton_hex, xi, eta, gamma, centroid_distance). + """ + elements_dir = gf_db_path / "elements" + candidates = [] + + for elem_dir in sorted(elements_dir.iterdir()): + if not elem_dir.is_dir(): + continue + coord_file = elem_dir / "coordinates.h5" + if not coord_file.exists(): + continue + + with h5py.File(coord_file, "r") as f: + cx = float(np.asarray(f.attrs["cx"]).flat[0]) + cy = float(np.asarray(f.attrs["cy"]).flat[0]) + cz = float(np.asarray(f.attrs["cz"]).flat[0]) + + dist = sqrt( + (cx - source_xyz[0]) ** 2 + + (cy - source_xyz[1]) ** 2 + + (cz - source_xyz[2]) ** 2 + ) + candidates.append((dist, elem_dir.name)) + + # Sort by centroid distance and check nearest candidates + candidates.sort() + + for dist, morton in candidates[:20]: # check top 20 nearest + coord_file = elements_dir / morton / "coordinates.h5" + with h5py.File(coord_file, "r") as f: + xyz_elem = f["xyz"][:] + + xi, eta, gamma, converged = locate_point_in_element(xyz_elem, source_xyz) + + if converged and all(abs(v) <= 1.01 for v in (xi, eta, gamma)): + return morton, xi, eta, gamma, dist + + # Fallback: return closest centroid (may not contain the point) + morton = candidates[0][1] + dist = candidates[0][0] + print(f" WARNING: no element contains the source, using closest centroid {morton}") + return morton, 0.0, 0.0, 0.0, dist + + +def butterworth_sos(order, fc, fs): + """Compute Butterworth lowpass SOS coefficients (matching Fortran implementation). + + Returns ndarray of shape (order//2, 6) with [b0, b1, b2, a0=1, a1, a2] per section. + """ + nsections = order // 2 + sos = np.zeros((nsections, 6)) + + # Pre-warp cutoff frequency + wc = 2.0 * fs * np.tan(pi * fc / fs) + wc2 = wc * wc + + for k in range(1, nsections + 1): + # Analog Butterworth pole angle + angle = pi * (2 * k + order - 1) / (2 * order) + pole_real = wc * cos(angle) + pole_imag = wc * sin(angle) + + # Bilinear transform coefficients + b0 = wc2 + b1 = 2.0 * wc2 + b2 = wc2 + + a0 = 4.0 * fs * fs - 4.0 * fs * pole_real + wc2 + a1 = 2.0 * wc2 - 8.0 * fs * fs + a2 = 4.0 * fs * fs + 4.0 * fs * pole_real + wc2 + + # Normalize + sos[k - 1] = [b0 / a0, b1 / a0, b2 / a0, 1.0, a1 / a0, a2 / a0] + + return sos + + +def sos_filter_forward(x, b0, b1, b2, a1, a2): + """Apply single SOS section forward (direct form II transposed).""" + y = np.zeros_like(x) + w1 = 0.0 + w2 = 0.0 + for i in range(len(x)): + yi = b0 * x[i] + w1 + w1 = b1 * x[i] - a1 * yi + w2 + w2 = b2 * x[i] - a2 * yi + y[i] = yi + return y + + +def sosfiltfilt(x, sos): + """Zero-phase SOS filter matching Fortran implementation. + + For each section: forward pass, then reverse + forward + reverse (backward pass). + This matches the Fortran gf_sosfiltfilt which processes one section at a time. + """ + y = x.copy() + for isec in range(sos.shape[0]): + b0, b1, b2, _, a1, a2 = sos[isec] + # Forward pass + y = sos_filter_forward(y, b0, b1, b2, a1, a2) + # Backward pass (reverse, filter, reverse) + y = y[::-1].copy() + y = sos_filter_forward(y, b0, b1, b2, a1, a2) + y = y[::-1].copy() + return y + + +def main(): + parser = argparse.ArgumentParser( + description="Cross-validate GF database against forward simulation" + ) + parser.add_argument( + "gf_db_path", type=Path, help="Path to GF database directory" + ) + parser.add_argument( + "forward_output", type=Path, help="Path to forward OUTPUT_FILES directory" + ) + parser.add_argument( + "--forward-solver-output", + type=Path, + default=None, + help="Path to forward output_solver.txt (default: /output_solver.txt)", + ) + parser.add_argument( + "--station", + type=str, + default="IU.SJG", + help="Station name as NET.STA (default: IU.SJG)", + ) + parser.add_argument( + "--gf-solver-output", + type=Path, + default=None, + help="Path to GF output_solver.txt (to get t0_gf; default: auto-detect)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Output plot path (default: gf_cross_validation.png next to forward output)", + ) + args = parser.parse_args() + + gf_db = args.gf_db_path + fwd_dir = args.forward_output + station = args.station + + if args.forward_solver_output: + solver_output = args.forward_solver_output + else: + solver_output = fwd_dir / "output_solver.txt" + + if args.output: + output_plot = Path(args.output) + else: + output_plot = fwd_dir.parent / "gf_cross_validation.png" + + # --------------------------------------------------------------- + # 1. Parse forward simulation output + # --------------------------------------------------------------- + print("Parsing forward solver output...") + src = parse_solver_output(solver_output) + print(f" Source (x, y, z): ({src['x']:.8f}, {src['y']:.8f}, {src['z']:.8f})") + print(f" Source (xi, eta, gamma): ({src['xi']:.8f}, {src['eta']:.8f}, {src['gamma']:.8f})") + print(f" nu1 (N): {src['nu1']}") + print(f" nu2 (E): {src['nu2']}") + print(f" nu3 (Z): {src['nu3']}") + + # Determine which force component was used in forward simulation + force_dir = np.array([src.get("force_E", 0), src.get("force_N", 0), src.get("force_Z", 0)]) + print(f" Force direction (E, N, Z): {force_dir}") + + # Build the force direction in Cartesian from nu vectors + # force_cart = force_E * nu_E + force_N * nu_N + force_Z * nu_Z + # where nu_N = nu1, nu_E = nu2, nu_Z = nu3 + nu_force = ( + src.get("force_N", 0) * src["nu1"] + + src.get("force_E", 0) * src["nu2"] + + src.get("force_Z", 0) * src["nu3"] + ) + print(f" Force direction (Cartesian): {nu_force}") + + # --------------------------------------------------------------- + # 2. Read mesh info + # --------------------------------------------------------------- + print("\nReading mesh info...") + with h5py.File(gf_db / "mesh_info.h5", "r") as f: + dt = float(np.asarray(f.attrs["dt"]).flat[0]) + nstep = int(np.asarray(f.attrs["nstep"]).flat[0]) + subsample_step = int(np.asarray(f.attrs["subsample_step"]).flat[0]) + nt_sub = int(np.asarray(f.attrs["nt_subsampled"]).flat[0]) + print(f" dt={dt}, nstep={nstep}, subsample_step={subsample_step}, nt_sub={nt_sub}") + + # Read station metadata for STF filter params + sta_file = gf_db / "stations" / f"{station}.h5" + with h5py.File(sta_file, "r") as f: + f_cutoff = float(np.asarray(f.attrs["f_cutoff"]).flat[0]) + hdur = float(np.asarray(f.attrs["hdur"]).flat[0]) + print(f" f_cutoff={f_cutoff:.4f} Hz, hdur={hdur:.6f} s") + + # --------------------------------------------------------------- + # 3. Find containing element + # --------------------------------------------------------------- + print("\nFinding containing element...") + source_xyz = np.array([src["x"], src["y"], src["z"]]) + morton_hex, xi, eta, gamma, dist = find_containing_element(gf_db, source_xyz) + print(f" Containing element: {morton_hex} (centroid dist: {dist:.6e})") + print(f" Located at (xi={xi:.6f}, eta={eta:.6f}, gamma={gamma:.6f})") + + # --------------------------------------------------------------- + # 4. Read GF displacement and interpolate + # --------------------------------------------------------------- + print("\nReading GF displacement...") + gf_file = gf_db / "elements" / morton_hex / f"{station}.h5" + with h5py.File(gf_file, "r") as f: + # Python shape: (nt_sub, NGLLZ, NGLLY, NGLLX, 3_disp, 3_force) + displ = f["displacement"][:] + print(f" Displacement shape: {displ.shape}, dtype: {displ.dtype}") + + # Interpolate to the exact source point using reference coordinates + print(f" Interpolating at (xi={xi:.6f}, eta={eta:.6f}, gamma={gamma:.6f})...") + + # displ shape: (nt, k, j, i, disp_comp, force_comp) + # We want to contract over k, j, i dimensions (indices 1, 2, 3) + # Result: G[nt, disp_comp, force_comp] + G = interpolate_gll(displ, xi, eta, gamma) + print(f" Green tensor shape: {G.shape}") # should be (nt_sub, 3, 3) + + # --------------------------------------------------------------- + # 5. Apply reciprocity + # --------------------------------------------------------------- + # G[t, disp_xyz, force_NEZ] is the Cartesian displacement at the source + # from a geographic force at the station. + # + # By reciprocity, the geographic displacement at the station from a + # Cartesian force F_cart at the source is: + # u_geo_b(station) = sum_j F_cart_j * G[t, j, b] + # + # Our forward force is in geographic direction -> Cartesian via nu: + # F_cart = force_E * nu_E + force_N * nu_N + force_Z * nu_Z = nu_force + # + # So: u_b(station) = nu_force . G[:, :, b] + + print("\nApplying reciprocity...") + gf_N = G[:, :, 0] @ nu_force # geographic N at station + gf_E = G[:, :, 1] @ nu_force # geographic E at station + gf_Z = G[:, :, 2] @ nu_force # geographic Z at station + print(f" GF trace ranges: N=[{gf_N.min():.6e}, {gf_N.max():.6e}], " + f"E=[{gf_E.min():.6e}, {gf_E.max():.6e}], " + f"Z=[{gf_Z.min():.6e}, {gf_Z.max():.6e}]") + + # --------------------------------------------------------------- + # 6. Convolve GF reconstruction with corrected STF + # --------------------------------------------------------------- + # The GF database has hdur_gf baked into the stored displacement. + # The forward simulation uses hdur_fwd (from f0 / SOURCE_DECAY_MIMIC_TRIANGLE). + # By Tarantola (2005, eq. 6.21), the convolution of two Gaussians gives a + # Gaussian with sigma_total = sqrt(sigma_1^2 + sigma_2^2). + # So we convolve the GF reconstruction with a corrected Gaussian: + # hdur_corrected = sqrt(hdur_fwd^2 - hdur_gf^2) + # to produce the equivalent of the forward trace. + + SOURCE_DECAY_MIMIC_TRIANGLE = 1.628 + + # Parse forward hdur from solver output + fwd_text = Path(solver_output).read_text() + m = re.search(r"Gaussian half duration:\s*([\d.Ee+-]+)", fwd_text) + if m: + hdur_fwd = float(m.group(1)) + else: + # Fallback: read f0 from FORCESOLUTION + m2 = re.search(r"half duration:\s*([\d.Ee+-]+)\s*seconds", fwd_text) + hdur_fwd = float(m2.group(1)) / SOURCE_DECAY_MIMIC_TRIANGLE if m2 else hdur + + hdur_gf = hdur # from station metadata + + print(f"\nSTF correction (Tarantola 2005, eq. 6.21):") + print(f" hdur_fwd (forward Gaussian): {hdur_fwd:.6f} s") + print(f" hdur_gf (GF database): {hdur_gf:.6f} s") + + if hdur_fwd > hdur_gf: + hdur_corrected = np.sqrt(hdur_fwd**2 - hdur_gf**2) + else: + print(" WARNING: hdur_fwd <= hdur_gf, no correction needed") + hdur_corrected = 0.0 + + print(f" hdur_corrected: {hdur_corrected:.6f} s") + + # Build corrected Gaussian STF and convolve with GF traces + # Time axis for GF (subsampled dt) + dt_sub = dt * subsample_step + # STF kernel: symmetric Gaussian centered at t=0 + # Extent: +-4*hdur_corrected should be sufficient + if hdur_corrected > 0: + stf_half_len = int(4.0 * hdur_corrected / dt_sub) + stf_t = np.arange(-stf_half_len, stf_half_len + 1) * dt_sub + stf = np.exp(-(stf_t / hdur_corrected)**2) / (np.sqrt(pi) * hdur_corrected) + stf *= dt_sub # normalize for discrete convolution + + print(f" STF kernel: {len(stf)} samples, peak={stf.max():.6e}") + + # Convolve GF traces with corrected STF using FFT for proper handling. + # The STF is causal-shifted: we want the peak of the Gaussian at t=0 + # to correspond to no time shift in the output. + from scipy.signal import fftconvolve + gf_N = fftconvolve(gf_N, stf, mode="same") + gf_E = fftconvolve(gf_E, stf, mode="same") + gf_Z = fftconvolve(gf_Z, stf, mode="same") + print(f" After convolution - GF trace ranges:") + print(f" N=[{gf_N.min():.6e}, {gf_N.max():.6e}]") + print(f" E=[{gf_E.min():.6e}, {gf_E.max():.6e}]") + print(f" Z=[{gf_Z.min():.6e}, {gf_Z.max():.6e}]") + + # --------------------------------------------------------------- + # 7. Read and filter forward seismograms + # --------------------------------------------------------------- + print("\nReading forward seismograms...") + fwd_traces = {} + for comp in ["MXN", "MXE", "MXZ"]: + sac_file = fwd_dir / f"{station}.{comp}.sem.sac" + tr = obspy_read(str(sac_file))[0] + time = tr.times() + tr.stats.sac.b # seconds from origin time + fwd_traces[comp] = {"time": time, "amp": tr.data.astype(np.float64)} + print(f" {comp}: {len(tr.data)} samples, " + f"amp range [{tr.data.min():.6e}, {tr.data.max():.6e}]") + + # Apply Butterworth filter to match GF anti-alias filtering, then subsample + print(f"\nApplying Butterworth lowpass filter (order=4, fc={f_cutoff:.4f} Hz, fs={1/dt:.4f} Hz)...") + sos = butterworth_sos(4, f_cutoff, 1.0 / dt) + + fwd_filtered = {} + for comp in ["MXN", "MXE", "MXZ"]: + amp = fwd_traces[comp]["amp"].astype(np.float64) + filtered = sosfiltfilt(amp, sos) + fwd_filtered[comp] = filtered + + # Subsample forward traces + fwd_sub = {} + for comp in ["MXN", "MXE", "MXZ"]: + fwd_sub[comp] = fwd_filtered[comp][::subsample_step] + time_fwd = fwd_traces["MXN"]["time"][::subsample_step] + + # Construct GF time axis + # GF stores at it = subsample_step, 2*subsample_step, ... + # Time for snapshot isnap (1-based): t = (isnap * subsample_step - 1) * dt - t0_gf + # Parse t0_gf from GF solver output + t0_gf = None + gf_solver_candidates = [ + args.gf_solver_output, + fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver.txt", + fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver_N.txt", + ] + for gf_solver_path in gf_solver_candidates: + if gf_solver_path is not None and gf_solver_path.exists(): + gf_solver_text = gf_solver_path.read_text() + m = re.search(r"start time\s*:\s*([-\dE.+]+)", gf_solver_text) + if m: + t0_gf = -float(m.group(1)) # start time is -t0 + print(f" Found GF t0 from: {gf_solver_path}") + break + if t0_gf is None: + # Fallback: t0_gf = T_min_period / 2 + t0_gf = hdur_gf * 5.0 # approximate + print(f" WARNING: could not find GF t0, estimating t0_gf={t0_gf:.4f}") + + dt_sub = dt * subsample_step + time_gf = np.array([(isnap * subsample_step - 1) * dt - t0_gf + for isnap in range(1, nt_sub + 1)]) + + print(f" Forward subsampled: {len(time_fwd)} samples, t=[{time_fwd[0]:.2f}, {time_fwd[-1]:.2f}]") + print(f" GF: {nt_sub} samples, t=[{time_gf[0]:.2f}, {time_gf[-1]:.2f}]") + print(f" t0_gf={t0_gf:.4f}, t0_fwd={-fwd_traces['MXN']['time'][0]:.4f}") + + # Align traces using overlapping time window + # Find common time range + t_start = max(time_fwd[0], time_gf[0]) + t_end = min(time_fwd[-1], time_gf[-1]) + print(f" Common time window: [{t_start:.2f}, {t_end:.2f}] s") + + # Use GF time axis as reference (it has fewer samples) + # Interpolate forward trace onto GF time points within the common window + mask_gf = (time_gf >= t_start) & (time_gf <= t_end) + time_common = time_gf[mask_gf] + + gf_traces = { + "MXN": gf_N[mask_gf], + "MXE": gf_E[mask_gf], + "MXZ": gf_Z[mask_gf], + } + + fwd_interp = {} + for comp in ["MXN", "MXE", "MXZ"]: + fwd_interp[comp] = np.interp(time_common, time_fwd, fwd_sub[comp]) + + n_compare = len(time_common) + print(f" Comparison samples: {n_compare}") + + # --------------------------------------------------------------- + # 8. Compute residuals + # --------------------------------------------------------------- + print("\nResiduals:") + for comp in ["MXN", "MXE", "MXZ"]: + fwd = fwd_interp[comp] + gf = gf_traces[comp] + residual = np.sqrt(np.sum((fwd - gf) ** 2)) + ref = np.sqrt(np.sum(fwd ** 2)) + rel_err = residual / ref if ref > 0 else float("inf") + print(f" {comp}: ||fwd - gf|| / ||fwd|| = {rel_err:.6e} " + f"(abs: {residual:.6e}, ref: {ref:.6e})") + + # --------------------------------------------------------------- + # 9. Plot comparison + # --------------------------------------------------------------- + print(f"\nPlotting to {output_plot}...") + fig, axes = plt.subplots(3, 1, figsize=(14, 10), sharex=True) + comp_labels = {"MXN": "North", "MXE": "East", "MXZ": "Vertical"} + + for ax, comp in zip(axes, ["MXN", "MXE", "MXZ"]): + fwd = fwd_interp[comp] + gf = gf_traces[comp] + + residual = np.sqrt(np.sum((fwd - gf) ** 2)) + ref = np.sqrt(np.sum(fwd ** 2)) + rel_err = residual / ref if ref > 0 else float("inf") + + ax.plot(time_common, fwd, "b-", linewidth=0.8, label="Forward (filtered)") + ax.plot(time_common, gf, "r--", linewidth=0.8, label="GF reconstructed") + ax.set_ylabel(f"{comp} ({comp_labels[comp]})") + ax.set_title(f"{comp} — relative error: {rel_err:.4e}") + ax.legend(loc="upper right", fontsize=8) + ax.grid(True, alpha=0.3) + + axes[-1].set_xlabel("Time (s)") + fig.suptitle( + f"GF Cross-Validation: {station}\n" + f"Source at element {morton_hex}\n" + f"(xi={xi:.4f}, eta={eta:.4f}, gamma={gamma:.4f})", + fontsize=11, + ) + plt.tight_layout() + plt.savefig(output_plot, dpi=150) + print(f"Saved {output_plot}") + plt.close() + + +if __name__ == "__main__": + main() From 46600520f69ed7696864bc8a399f5df874f375ec Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Sun, 17 May 2026 16:37:05 -0400 Subject: [PATCH 09/20] Fixing topography writing --- src/specfem3D/green_function_metadata.F90 | 21 +++++++++++++++---- .../prepare_green_function_storage.f90 | 9 +++++++- src/specfem3D/setup_sources_receivers.f90 | 3 ++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/specfem3D/green_function_metadata.F90 b/src/specfem3D/green_function_metadata.F90 index e8f8aad3e..3275e1df9 100644 --- a/src/specfem3D/green_function_metadata.F90 +++ b/src/specfem3D/green_function_metadata.F90 @@ -248,13 +248,26 @@ subroutine gf_write_mesh_info() filepath = trim(GF_DATABASE_PATH) // '/mesh_info.h5' - ! idempotent: skip if file already exists + ! idempotent: skip if file already exists and is valid inquire(file=trim(filepath), exist=file_exists) if (file_exists) then - write(IMAIN,*) 'Green function database: mesh_info.h5 already exists, skipping' - write(IMAIN,*) + ! validate by trying to open and read a known attribute + call h5fopen_f(trim(filepath), H5F_ACC_RDONLY_F, fid, hdferr) + if (hdferr == 0) then + call h5aopen_f(fid, 'scale_displ', attr_id, hdferr) + if (hdferr == 0) then + call h5aclose_f(attr_id, hdferr) + call h5fclose_f(fid, hdferr) + write(IMAIN,*) 'Green function database: mesh_info.h5 already exists, skipping' + write(IMAIN,*) + call flush_IMAIN() + return + endif + call h5fclose_f(fid, hdferr) + endif + ! file exists but is corrupt/incomplete — remove and recreate + write(IMAIN,*) 'Green function database: mesh_info.h5 is corrupt, recreating' call flush_IMAIN() - return endif ! create file diff --git a/src/specfem3D/prepare_green_function_storage.f90 b/src/specfem3D/prepare_green_function_storage.f90 index 81dfe69df..e8883c415 100644 --- a/src/specfem3D/prepare_green_function_storage.f90 +++ b/src/specfem3D/prepare_green_function_storage.f90 @@ -30,7 +30,9 @@ subroutine prepare_green_function_storage() ! Central wrapper for all Green function database preparation steps. ! Called from prepare_timerun() before coordinate arrays are deallocated. - use shared_parameters, only: GF_DATABASE_ENABLED + use shared_parameters, only: GF_DATABASE_ENABLED, TOPOGRAPHY + + use specfem_par, only: ibathy_topo implicit none @@ -51,6 +53,11 @@ subroutine prepare_green_function_storage() ! write shared mesh info file (topography + ellipticity, rank 0, idempotent) call gf_write_mesh_info() + ! ibathy_topo was kept alive for gf_write_mesh_info — deallocate now + if (TOPOGRAPHY) then + if (allocated(ibathy_topo)) deallocate(ibathy_topo) + endif + ! write per-station metadata file (rank 0, create-or-skip) call gf_write_station_metadata() diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index 8590a80b3..325e7d1a1 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -82,7 +82,8 @@ subroutine setup_sources_receivers() endif ! topography array no more needed - if (TOPOGRAPHY) then + ! (keep it allocated if GF database is enabled — gf_write_mesh_info needs it) + if (TOPOGRAPHY .and. .not. GF_DATABASE_ENABLED) then if (allocated(ibathy_topo) ) deallocate(ibathy_topo) endif From 50482b3a8d8ebe79e7881b6d1d4acf8ff607792a Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Tue, 19 May 2026 14:47:29 -0400 Subject: [PATCH 10/20] Updated the cross validation code. --- utils/green_function/gf_cross_validate.py | 144 +++++++++++++++++++--- 1 file changed, 128 insertions(+), 16 deletions(-) diff --git a/utils/green_function/gf_cross_validate.py b/utils/green_function/gf_cross_validate.py index 6f36de54a..1ea644feb 100644 --- a/utils/green_function/gf_cross_validate.py +++ b/utils/green_function/gf_cross_validate.py @@ -283,6 +283,39 @@ def find_containing_element(gf_db_path, source_xyz): return morton, 0.0, 0.0, 0.0, dist +def butterworth_highpass_sos(order, fc, fs): + """Compute Butterworth highpass SOS coefficients. + + Returns ndarray of shape (order//2, 6) with [b0, b1, b2, a0=1, a1, a2] per section. + """ + nsections = order // 2 + sos = np.zeros((nsections, 6)) + + # Pre-warp cutoff frequency + wc = 2.0 * fs * np.tan(pi * fc / fs) + + for k in range(1, nsections + 1): + # Analog Butterworth pole angle + angle = pi * (2 * k + order - 1) / (2 * order) + pole_real = wc * cos(angle) + + # Bilinear transform: highpass has s^2 numerator in analog domain + # H_hp(s) = s^2 / (s^2 - 2*Re(p)*s + |p|^2) for each conjugate pair + # After bilinear transform z = (1 + s/(2fs)) / (1 - s/(2fs)): + b0 = 4.0 * fs * fs + b1 = -8.0 * fs * fs + b2 = 4.0 * fs * fs + + a0 = 4.0 * fs * fs - 4.0 * fs * pole_real + wc * wc + a1 = 2.0 * wc * wc - 8.0 * fs * fs + a2 = 4.0 * fs * fs + 4.0 * fs * pole_real + wc * wc + + # Normalize + sos[k - 1] = [b0 / a0, b1 / a0, b2 / a0, 1.0, a1 / a0, a2 / a0] + + return sos + + def butterworth_sos(order, fc, fs): """Compute Butterworth lowpass SOS coefficients (matching Fortran implementation). @@ -347,6 +380,32 @@ def sosfiltfilt(x, sos): return y +def filter_with_taper(x, sos, taper_fraction=0.05, pad_factor=1.0): + """Taper signal edges to zero, zero-pad, filter, and trim. + + The taper brings the signal smoothly to zero at both ends so the + transition to zero-padding is seamless. The padding then absorbs + IIR filter edge transients. After filtering, the result is trimmed + back to the original length. + """ + n = len(x) + n_taper = max(1, int(taper_fraction * n)) + n_pad = int(pad_factor * n) + + # Cosine taper to bring edges to zero + taper = np.ones(n) + taper[:n_taper] = 0.5 * (1.0 - np.cos(np.linspace(0, pi, n_taper))) + taper[-n_taper:] = 0.5 * (1.0 - np.cos(np.linspace(pi, 0, n_taper))) + tapered = x * taper + + # Zero-pad on both sides + padded = np.concatenate([np.zeros(n_pad), tapered, np.zeros(n_pad)]) + + # Filter and trim back to original length + filtered = sosfiltfilt(padded, sos) + return filtered[n_pad:n_pad + n] + + def main(): parser = argparse.ArgumentParser( description="Cross-validate GF database against forward simulation" @@ -381,6 +440,20 @@ def main(): default=None, help="Output plot path (default: gf_cross_validation.png next to forward output)", ) + parser.add_argument( + "--highpass-period", + type=float, + default=None, + help="Apply zero-phase Butterworth highpass filter at this period (in seconds). " + "Removes energy at periods longer than this value.", + ) + parser.add_argument( + "--lowpass-period", + type=float, + default=None, + help="Override the lowpass filter period (in seconds). " + "Default uses the GF database anti-alias cutoff frequency.", + ) args = parser.parse_args() gf_db = args.gf_db_path @@ -558,7 +631,7 @@ def main(): # --------------------------------------------------------------- print("\nReading forward seismograms...") fwd_traces = {} - for comp in ["MXN", "MXE", "MXZ"]: + for comp in ["BXN", "BXE", "BXZ"]: sac_file = fwd_dir / f"{station}.{comp}.sem.sac" tr = obspy_read(str(sac_file))[0] time = tr.times() + tr.stats.sac.b # seconds from origin time @@ -567,20 +640,23 @@ def main(): f"amp range [{tr.data.min():.6e}, {tr.data.max():.6e}]") # Apply Butterworth filter to match GF anti-alias filtering, then subsample - print(f"\nApplying Butterworth lowpass filter (order=4, fc={f_cutoff:.4f} Hz, fs={1/dt:.4f} Hz)...") - sos = butterworth_sos(4, f_cutoff, 1.0 / dt) + lp_fc = 1.0 / args.lowpass_period if args.lowpass_period else f_cutoff + lp_filter = args.lowpass_period is not None + print(f"\nApplying Butterworth lowpass filter (order=4, fc={lp_fc:.4f} Hz" + f" [T={1/lp_fc:.1f}s], fs={1/dt:.4f} Hz)...") + sos = butterworth_sos(4, lp_fc, 1.0 / dt) fwd_filtered = {} - for comp in ["MXN", "MXE", "MXZ"]: + for comp in ["BXN", "BXE", "BXZ"]: amp = fwd_traces[comp]["amp"].astype(np.float64) - filtered = sosfiltfilt(amp, sos) + filtered = filter_with_taper(amp, sos) fwd_filtered[comp] = filtered # Subsample forward traces fwd_sub = {} - for comp in ["MXN", "MXE", "MXZ"]: + for comp in ["BXN", "BXE", "BXZ"]: fwd_sub[comp] = fwd_filtered[comp][::subsample_step] - time_fwd = fwd_traces["MXN"]["time"][::subsample_step] + time_fwd = fwd_traces["BXN"]["time"][::subsample_step] # Construct GF time axis # GF stores at it = subsample_step, 2*subsample_step, ... @@ -611,7 +687,7 @@ def main(): print(f" Forward subsampled: {len(time_fwd)} samples, t=[{time_fwd[0]:.2f}, {time_fwd[-1]:.2f}]") print(f" GF: {nt_sub} samples, t=[{time_gf[0]:.2f}, {time_gf[-1]:.2f}]") - print(f" t0_gf={t0_gf:.4f}, t0_fwd={-fwd_traces['MXN']['time'][0]:.4f}") + print(f" t0_gf={t0_gf:.4f}, t0_fwd={-fwd_traces['BXN']['time'][0]:.4f}") # Align traces using overlapping time window # Find common time range @@ -625,23 +701,54 @@ def main(): time_common = time_gf[mask_gf] gf_traces = { - "MXN": gf_N[mask_gf], - "MXE": gf_E[mask_gf], - "MXZ": gf_Z[mask_gf], + "BXN": gf_N[mask_gf], + "BXE": gf_E[mask_gf], + "BXZ": gf_Z[mask_gf], } fwd_interp = {} - for comp in ["MXN", "MXE", "MXZ"]: + for comp in ["BXN", "BXE", "BXZ"]: fwd_interp[comp] = np.interp(time_common, time_fwd, fwd_sub[comp]) n_compare = len(time_common) print(f" Comparison samples: {n_compare}") + # --------------------------------------------------------------- + # 7b. Optional lowpass filter on GF traces + # --------------------------------------------------------------- + # The forward traces are always lowpass-filtered before subsampling (section 7). + # The GF traces already have the database anti-alias filter baked in. + # When --lowpass-period is explicitly set, apply the same lowpass to GF traces + # so both are filtered identically. + if lp_filter: + lp_fc_sub = 1.0 / args.lowpass_period + lp_fs_sub = 1.0 / dt_sub + lp_order = 4 + print(f"\nApplying Butterworth lowpass filter to GF traces (order={lp_order}, " + f"period={args.lowpass_period:.1f} s, fc={lp_fc_sub:.6f} Hz, fs={lp_fs_sub:.4f} Hz)...") + lp_sos_sub = butterworth_sos(lp_order, lp_fc_sub, lp_fs_sub) + for comp in ["BXN", "BXE", "BXZ"]: + gf_traces[comp] = filter_with_taper(gf_traces[comp], lp_sos_sub) + + # --------------------------------------------------------------- + # 7c. Optional highpass filter + # --------------------------------------------------------------- + if args.highpass_period is not None: + hp_fc = 1.0 / args.highpass_period # convert period to frequency + hp_fs = 1.0 / dt_sub # sampling rate of subsampled traces + hp_order = 4 + print(f"\nApplying Butterworth highpass filter (order={hp_order}, " + f"period={args.highpass_period:.1f} s, fc={hp_fc:.6f} Hz, fs={hp_fs:.4f} Hz)...") + hp_sos = butterworth_highpass_sos(hp_order, hp_fc, hp_fs) + for comp in ["BXN", "BXE", "BXZ"]: + gf_traces[comp] = filter_with_taper(gf_traces[comp], hp_sos) + fwd_interp[comp] = filter_with_taper(fwd_interp[comp], hp_sos) + # --------------------------------------------------------------- # 8. Compute residuals # --------------------------------------------------------------- print("\nResiduals:") - for comp in ["MXN", "MXE", "MXZ"]: + for comp in ["BXN", "BXE", "BXZ"]: fwd = fwd_interp[comp] gf = gf_traces[comp] residual = np.sqrt(np.sum((fwd - gf) ** 2)) @@ -655,9 +762,9 @@ def main(): # --------------------------------------------------------------- print(f"\nPlotting to {output_plot}...") fig, axes = plt.subplots(3, 1, figsize=(14, 10), sharex=True) - comp_labels = {"MXN": "North", "MXE": "East", "MXZ": "Vertical"} + comp_labels = {"BXN": "North", "BXE": "East", "BXZ": "Vertical"} - for ax, comp in zip(axes, ["MXN", "MXE", "MXZ"]): + for ax, comp in zip(axes, ["BXN", "BXE", "BXZ"]): fwd = fwd_interp[comp] gf = gf_traces[comp] @@ -673,8 +780,13 @@ def main(): ax.grid(True, alpha=0.3) axes[-1].set_xlabel("Time (s)") + filter_info = "" + if args.lowpass_period: + filter_info += f", lowpass T<{args.lowpass_period:.0f}s" + if args.highpass_period: + filter_info += f", highpass T>{args.highpass_period:.0f}s" fig.suptitle( - f"GF Cross-Validation: {station}\n" + f"GF Cross-Validation: {station}{filter_info}\n" f"Source at element {morton_hex}\n" f"(xi={xi:.4f}, eta={eta:.4f}, gamma={gamma:.4f})", fontsize=11, From 5551ef8bff3c0b89f14ecda16dda834be034062a Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Tue, 2 Jun 2026 11:18:06 -0400 Subject: [PATCH 11/20] Added regional example to the EXAMPLES directory. --- .gitignore | 3 + .../regional/.gitignore | 4 + .../regional/Snakefile | 469 ++++++++++++++++++ .../regional/db_base/DATA/FORCESOLUTION | 11 + .../regional/db_base/DATA/GF_LOCATIONS | 8 + .../regional/db_base/DATA/Par_file | 450 +++++++++++++++++ .../regional/db_base/DATA/STATIONS | 1 + .../regional/validation_data/CMTSOLUTION | 13 + .../regional/validation_data/FORCESOLUTION | 11 + .../regional/validation_data/STATIONS | 18 + src/specfem3D/green_function_metadata.F90 | 46 ++ 11 files changed, 1034 insertions(+) create mode 100644 EXAMPLES/green_function_database/regional/.gitignore create mode 100644 EXAMPLES/green_function_database/regional/Snakefile create mode 100644 EXAMPLES/green_function_database/regional/db_base/DATA/FORCESOLUTION create mode 100644 EXAMPLES/green_function_database/regional/db_base/DATA/GF_LOCATIONS create mode 100644 EXAMPLES/green_function_database/regional/db_base/DATA/Par_file create mode 100644 EXAMPLES/green_function_database/regional/db_base/DATA/STATIONS create mode 100644 EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION create mode 100644 EXAMPLES/green_function_database/regional/validation_data/FORCESOLUTION create mode 100644 EXAMPLES/green_function_database/regional/validation_data/STATIONS diff --git a/.gitignore b/.gitignore index 96b367409..74309c44d 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ doc/USER_MANUAL/figures/*-eps-converted-to.pdf # slurm* submit* + +# Mac directory file +.DS_Store \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/.gitignore b/EXAMPLES/green_function_database/regional/.gitignore new file mode 100644 index 000000000..143bf66b2 --- /dev/null +++ b/EXAMPLES/green_function_database/regional/.gitignore @@ -0,0 +1,4 @@ +.snakemake +db_base/DATABASES_MPI +db_base/OUTPUT_FILES +db_base/mesher.flag \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/Snakefile b/EXAMPLES/green_function_database/regional/Snakefile new file mode 100644 index 000000000..74f825228 --- /dev/null +++ b/EXAMPLES/green_function_database/regional/Snakefile @@ -0,0 +1,469 @@ +""" +Snakemake workflow for Green Function database generation. + +For each station in the STATIONS file, runs 3 reciprocal simulations (N, E, Z force +components) sequentially from the same directory, then builds the manifest. + +All simulations write to a shared GF_DATABASE_PATH (absolute path from Par_file). +The 3 components per station are sequential so each writes its force-component slice +into the same HDF5 files without conflict. + +Usage: + snakemake -j1 # fully sequential + snakemake -jN --resources mpi=1 # N stations in parallel (components sequential) + +Configuration (edit below or override via --config): + SPECFEM_DIR: path to specfem3d_globe repository root (default: ../../..) + BASEDIR: path to base simulation template (contains DATA/Par_file, DATA/STATIONS) + STATIONS: path to STATIONS file + NPROC: number of MPI ranks + MPIRUN: MPI launcher command + CREATE_VALIDATION: True/False — create a forward simulation directory for validation + VALIDATION_DATA: path to validation data (STATIONS, CMTSOLUTION, FORCESOLUTION) + VALIDATION_DIR: output directory for the validation forward simulation +""" + +import os +import shutil + +# ─── Configuration ─────────────────────────────────────────────────────────── + +# Path to the specfem3d_globe repository root (for binaries and DATA files) +SPECFEM_DIR = config.get("SPECFEM_DIR", os.path.join("..", "..", "..")) + +# Base directory containing the meshed model (bin/, DATABASES_MPI/, DATA/Par_file) +BASEDIR = config.get("BASEDIR", "db_base") + +# Stations file +STATIONS_FILE = config.get("STATIONS", os.path.join(BASEDIR, "DATA", "STATIONS")) + +# MPI configuration +NPROC = int(config.get("NPROC", 4)) +MPIRUN = config.get("MPIRUN", "mpirun") + +# Force source parameters +# Note: f0 is NOT used for GF STF computation (gf_hdur is derived from T_min_period). +# It only affects the initial source half-duration for time-step stability checks. +F0 = config.get("F0", "0.0500") +FACTOR_FORCE = config.get("FACTOR_FORCE", "1.0d15") + +# Output root for all station simulations +SIMDIR = config.get("SIMDIR", "simulations") + +# Validation forward simulation toggle +CREATE_VALIDATION = config.get("CREATE_VALIDATION", True) +VALIDATION_DATA = config.get("VALIDATION_DATA", "validation_data") +VALIDATION_DIR = config.get("VALIDATION_DIR", "forward") + + +# ─── GF database output path ───────────────────────────────────────────────── + +# Default: GFDB/ inside the example directory (resolved to absolute path) +GF_DATABASE_PATH = os.path.abspath(config.get("GF_DATABASE_PATH", "GFDB")) + + +# ─── Parse STATIONS file ───────────────────────────────────────────────────── + +def parse_stations(stations_file): + """Parse SPECFEM STATIONS file.""" + stations = [] + with open(stations_file) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) >= 6: + stations.append({ + "station": parts[0], + "network": parts[1], + "latitude": parts[2], + "longitude": parts[3], + "elevation": parts[4], + "burial": parts[5], + }) + return stations + + +STATIONS = parse_stations(STATIONS_FILE) +STATION_IDS = [f"{s['network']}.{s['station']}" for s in STATIONS] +COMPONENTS = ["N", "E", "Z"] + + +# ─── Helper functions ──────────────────────────────────────────────────────── + +def get_station_info(station_id): + """Look up station metadata by network.station ID.""" + for s in STATIONS: + if f"{s['network']}.{s['station']}" == station_id: + return s + raise ValueError(f"Station {station_id} not found") + + +def forcesolution_content(station_id, comp): + """Generate FORCESOLUTION file content for a given station and component.""" + s = get_station_info(station_id) + comp_n = "1.0" if comp == "N" else "0.0" + comp_e = "1.0" if comp == "E" else "0.0" + comp_z = "1.0" if comp == "Z" else "0.0" + + return ( + f"FORCE {station_id}\n" + f"time shift: 0.0000\n" + f"f0: {F0}\n" + f"latitude: {s['latitude']}\n" + f"longitude: {s['longitude']}\n" + f"depth: {float(s['burial']):.4f}\n" + f"source time function: 0\n" + f"factor force source: {FACTOR_FORCE}\n" + f"comp dir vect source E: {comp_e}\n" + f"comp dir vect source N: {comp_n}\n" + f"comp dir vect source Z: {comp_z}\n" + ) + + +# ─── Rules ─────────────────────────────────────────────────────────────────── + +def all_inputs(wildcards): + """Collect all target files for the workflow.""" + inputs = [] + # All Z simulations done (Z runs last, so implies N and E are done too) + inputs += expand(os.path.join(SIMDIR, "{station}", "done_Z.flag"), + station=STATION_IDS) + # Manifest built + inputs.append(os.path.join(GF_DATABASE_PATH, "centroids.bin")) + # Optionally run validation forward simulation + if CREATE_VALIDATION: + inputs.append(os.path.join(VALIDATION_DIR, "done.flag")) + return inputs + + +rule all: + input: + all_inputs, + + +rule setup_base: + """Set up the base directory: link specfem binaries and data, then run the mesher.""" + output: + flag=os.path.join(BASEDIR, "mesher.flag"), + params: + basedir=BASEDIR, + specfem_dir=SPECFEM_DIR, + gf_database_path=GF_DATABASE_PATH, + nproc=NPROC, + mpirun=MPIRUN, + run: + import re + + basedir = params.basedir + specfem_dir = os.path.abspath(params.specfem_dir) + + # Create required directories + os.makedirs(os.path.join(basedir, "DATABASES_MPI"), exist_ok=True) + os.makedirs(os.path.join(basedir, "OUTPUT_FILES"), exist_ok=True) + os.makedirs(params.gf_database_path, exist_ok=True) + + # Symlink bin/ to specfem binaries + bin_link = os.path.join(basedir, "bin") + bin_target = os.path.join(specfem_dir, "bin") + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + + # Symlink required DATA subdirectories + for name in ["topo_bathy", "s362ani", "crust2.0"]: + link = os.path.join(basedir, "DATA", name) + target = os.path.join(specfem_dir, "DATA", name) + if not os.path.exists(link): + os.symlink(target, link) + + # Update GF_DATABASE_PATH in Par_file to match the configured path + parfile = os.path.join(basedir, "DATA", "Par_file") + with open(parfile, "r") as f: + content = f.read() + content = re.sub( + r"^(GF_DATABASE_PATH\s*=\s*).*$", + rf"\g<1>{params.gf_database_path}/", + content, + flags=re.MULTILINE, + ) + with open(parfile, "w") as f: + f.write(content) + + # Run the mesher + shell( + "cd {basedir} && " + "{params.mpirun} -np {params.nproc} ./bin/xmeshfem3D " + "> OUTPUT_FILES/output_mesher.txt 2>&1" + ) + + # Touch flag + with open(output.flag, "w") as f: + f.write("mesher complete\n") + + +rule setup_station: + """Create simulation directory for a station (shared by all 3 components).""" + input: + base=os.path.join(BASEDIR, "mesher.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "setup.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + basedir=BASEDIR, + stations_file=STATIONS_FILE, + run: + simdir = params.simdir + + # Create directories + os.makedirs(os.path.join(simdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(simdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ + bin_link = os.path.join(simdir, "bin") + db_link = os.path.join(simdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file + shutil.copy2( + os.path.join(params.basedir, "DATA", "Par_file"), + os.path.join(simdir, "DATA", "Par_file"), + ) + + # Copy STATIONS file + shutil.copy2(params.stations_file, os.path.join(simdir, "DATA", "STATIONS")) + + # Copy GF_LOCATIONS if it exists + gf_loc_src = os.path.join(params.basedir, "DATA", "GF_LOCATIONS") + if os.path.exists(gf_loc_src): + shutil.copy2(gf_loc_src, os.path.join(simdir, "DATA", "GF_LOCATIONS")) + + # Touch flag + with open(output.flag, "w") as f: + f.write("setup complete\n") + + +rule run_component_N: + """Run N-component simulation (first).""" + input: + setup=os.path.join(SIMDIR, "{station}", "setup.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_N.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for N component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "N")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_N.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("N complete\n") + + +rule run_component_E: + """Run E-component simulation (after N).""" + input: + prev=os.path.join(SIMDIR, "{station}", "done_N.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_E.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for E component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "E")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_E.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("E complete\n") + + +rule run_component_Z: + """Run Z-component simulation (after E).""" + input: + prev=os.path.join(SIMDIR, "{station}", "done_E.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_Z.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for Z component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "Z")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_Z.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("Z complete\n") + + +rule build_manifest: + """Build centroids.bin and manifest.csv after all stations complete.""" + input: + expand(os.path.join(SIMDIR, "{station}", "done_Z.flag"), + station=STATION_IDS), + output: + centroids=os.path.join(GF_DATABASE_PATH, "centroids.bin"), + manifest_csv=os.path.join(GF_DATABASE_PATH, "manifest.csv"), + params: + gf_db_path=GF_DATABASE_PATH, + script="/Users/lsawade/specfem3d_globe/utils/green_function/gf_build_manifest.py", + shell: + """ + python {params.script} {params.gf_db_path} + """ + +rule setup_validation_forward: + """Set up a forward simulation directory for validation using data from validation_data/.""" + input: + # Only run after the GF database is built + os.path.join(GF_DATABASE_PATH, "centroids.bin"), + output: + flag=os.path.join(VALIDATION_DIR, "setup.flag"), + params: + basedir=BASEDIR, + validation_data=VALIDATION_DATA, + validation_dir=VALIDATION_DIR, + run: + import re + + vdir = params.validation_dir + + # Create directories + os.makedirs(os.path.join(vdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(vdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ from base directory + bin_link = os.path.join(vdir, "bin") + db_link = os.path.join(vdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file from base directory and disable GF_DATABASE_ENABLED + parfile_src = os.path.join(params.basedir, "DATA", "Par_file") + parfile_dst = os.path.join(vdir, "DATA", "Par_file") + with open(parfile_src, "r") as f: + parfile_content = f.read() + parfile_content = re.sub( + r"^(\s*GF_DATABASE_ENABLED\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + with open(parfile_dst, "w") as f: + f.write(parfile_content) + + # Copy STATIONS from validation_data + shutil.copy2( + os.path.join(params.validation_data, "STATIONS"), + os.path.join(vdir, "DATA", "STATIONS"), + ) + + # Copy CMTSOLUTION if it exists + cmt_src = os.path.join(params.validation_data, "CMTSOLUTION") + if os.path.exists(cmt_src): + shutil.copy2(cmt_src, os.path.join(vdir, "DATA", "CMTSOLUTION")) + + # Copy FORCESOLUTION if it exists + force_src = os.path.join(params.validation_data, "FORCESOLUTION") + if os.path.exists(force_src): + shutil.copy2(force_src, os.path.join(vdir, "DATA", "FORCESOLUTION")) + + # Touch flag + with open(output.flag, "w") as f: + f.write("validation forward setup complete\n") + + +rule run_validation_forward: + """Run the validation forward simulation.""" + input: + setup=os.path.join(VALIDATION_DIR, "setup.flag"), + output: + flag=os.path.join(VALIDATION_DIR, "done.flag"), + params: + validation_dir=VALIDATION_DIR, + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + vdir = params.validation_dir + shell( + "cd {vdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver.txt 2>&1" + ) + with open(output.flag, "w") as f: + f.write("validation forward complete\n") + + +rule clean_base: + """Clean base directory links and mesher output.""" + shell: + "rm -rf {BASEDIR}/bin {BASEDIR}/DATABASES_MPI {BASEDIR}/OUTPUT_FILES " + "{BASEDIR}/DATA/topo_bathy {BASEDIR}/DATA/s362ani {BASEDIR}/DATA/crust2.0 " + "{BASEDIR}/mesher.flag" + +rule clean_simulations: + """Clean all simulation directories and output files.""" + shell: + "rm -rf {SIMDIR}/*" + +rule clean_validation: + """Clean validation forward simulation directory.""" + shell: + "rm -rf {VALIDATION_DIR}/*" + +rule clean_database: + """Clean GF database output files.""" + shell: + "rm -rf {GF_DATABASE_PATH}/*" + +rule clean_all: + """Clean base, simulations, validation, and database outputs.""" + shell: + "rm -rf {BASEDIR}/bin {BASEDIR}/DATABASES_MPI {BASEDIR}/OUTPUT_FILES " + "{BASEDIR}/DATA/topo_bathy {BASEDIR}/DATA/s362ani {BASEDIR}/DATA/crust2.0 " + "{BASEDIR}/mesher.flag " + "{SIMDIR}/* {GF_DATABASE_PATH}/* {VALIDATION_DIR}/*" \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/FORCESOLUTION b/EXAMPLES/green_function_database/regional/db_base/DATA/FORCESOLUTION new file mode 100644 index 000000000..95047dab5 --- /dev/null +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/FORCESOLUTION @@ -0,0 +1,11 @@ +FORCE IU.SJG +time shift: 0.0000 +f0: 0.0500 +latitude: -13.8200 +longitude: -67.2500 +depth: 647.1000 +source time function: 0 +factor force source: 1.0d15 +comp dir vect source E: 0.0 +comp dir vect source N: 0.0 +comp dir vect source Z: 1.0 diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/GF_LOCATIONS b/EXAMPLES/green_function_database/regional/db_base/DATA/GF_LOCATIONS new file mode 100644 index 000000000..c1ea83fac --- /dev/null +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/GF_LOCATIONS @@ -0,0 +1,8 @@ +# GF_LOCATIONS - earthquake hypocenter locations for Green function database +# latitude longitude depth_km +# 2010 Chile earthquake (Maule) +-35.909 -72.733 35.0 +# 2019 Peru earthquake +-5.812 -75.270 122.6 +# Near center of chunk +-13.0 -67.0 10.0 diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file new file mode 100644 index 000000000..5f9eeaaca --- /dev/null +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file @@ -0,0 +1,450 @@ +#----------------------------------------------------------- +# +# Simulation input parameters +# +#----------------------------------------------------------- + +# forward or adjoint simulation +SIMULATION_TYPE = 1 # set to 1 for forward simulations, 2 for adjoint simulations for sources, and 3 for kernel simulations +NOISE_TOMOGRAPHY = 0 # flag of noise tomography, three steps (1,2,3). If earthquake simulation, set it to 0. +SAVE_FORWARD = .false. # save last frame of forward simulation or not + +# number of chunks (1,2,3 or 6) +NCHUNKS = 1 + +# angular width of the first chunk (not used if full sphere with six chunks) +ANGULAR_WIDTH_XI_IN_DEGREES = 90.d0 # angular size of a chunk +ANGULAR_WIDTH_ETA_IN_DEGREES = 90.d0 +CENTER_LATITUDE_IN_DEGREES = -13.8200d0 +CENTER_LONGITUDE_IN_DEGREES = -67.2500 +GAMMA_ROTATION_AZIMUTH = 0.d0 + +# number of elements at the surface along the two sides of the first chunk +# (must be multiple of 16 and 8 * multiple of NPROC below) +NEX_XI = 64 +NEX_ETA = 64 + +# number of MPI processors along the two sides of the first chunk +NPROC_XI = 2 +NPROC_ETA = 2 + +#----------------------------------------------------------- +# +# Model +# +#----------------------------------------------------------- + +# 1D models with real structure: +# 1D_isotropic_prem, 1D_transversely_isotropic_prem, 1D_iasp91, 1D_1066a, 1D_ak135f_no_mud, 1D_ref, 1D_ref_iso, 1D_jp3d, 1D_sea99, 1D_Berkeley +# +# 1D models with only one fictitious averaged crustal layer: +# 1D_isotropic_prem_onecrust, 1D_transversely_isotropic_prem_onecrust, 1D_iasp91_onecrust, 1D_1066a_onecrust, 1D_ak135f_no_mud_onecrust +# +# fully 3D models: +# transversely_isotropic_prem_plus_3D_crust_2.0, 3D_anisotropic, 3D_attenuation, +# s20rts, s40rts, s362ani, s362iso, s362wmani, s362ani_prem, s362ani_3DQ, s362iso_3DQ, +# s29ea, sea99_jp3d1994, sea99, jp3d1994, heterogen, full_sh, sgloberani_aniso, sgloberani_iso, +# spiral, emc_model, emc_model_qmu, emc_model_tiso, emc_model_tiso_qmu, semucb_A3d, semucb_A3d_3dQ +# (see manual for more...) +# +# 3D crustal models: +# crust1.0, crust2.0, EPcrust, EuCRUST, crustmaps, crustSH +# +# Mars models: +# 1D_Sohl, 1D_Sohl_3D_crust, 1D_case65TAY, 1D_case65TAY_3D_crust, mars_1D, mars_1D_3D_crust +# +# Moon models: +# vpremoon +# +# 3D models with 3D crust: append "_**crustname**" to the mantle model name +# to take a 3D crustal model (by default crust2.0 is taken for 3D mantle models) +# e.g. s20rts_crust1.0, s362ani_crustmaps, full_sh_crustSH, sglobe_EPcrust, etc. +# +# 3D models with 1D crust: append "_1Dcrust" to the 3D model name +# to take the 1D crustal model from the +# associated reference model rather than the default 3D crustal model +# e.g. s20rts_1Dcrust, s362ani_1Dcrust, etc. +# +MODEL = s362ani + +# parameters describing the Earth model +OCEANS = .true. +ELLIPTICITY = .true. +TOPOGRAPHY = .true. +GRAVITY = .true. +ROTATION = .true. +ATTENUATION = .true. + +# full gravity calculation by solving Poisson's equation for gravity potential instead of using a Cowling approximation +# (must have also GRAVITY flag set to .true. to become active) +FULL_GRAVITY = .false. +# for full gravity calculation, set to 0 == builtin or 1 == PETSc Poisson solver +# (the PETSc solver option needs the PETSc library installed; code configuration --with-petsc) +POISSON_SOLVER = 0 + +# record length in minutes +RECORD_LENGTH_IN_MINUTES = 30.0d0 + +#----------------------------------------------------------- +# +# Mesh +# +#----------------------------------------------------------- + +## regional mesh cut-off +# using this flag will cut-off the mesh in the mantle at a layer matching to the given cut-off depth. +# this flag only has an effect for regional simulations, i.e., for NCHUNKS values less than 6. +REGIONAL_MESH_CUTOFF = .true. + +# regional mesh cut-off depth (in km) +# possible selections are: 24.4d0, 80.d0, 220.d0, 400.d0, 600.d0, 670.d0, 771.d0 +REGIONAL_MESH_CUTOFF_DEPTH = 771.d0 + +# regional mesh cut-off w/ a second doubling layer below 220km interface +# (by default, a first doubling layer will be added below the Moho, and a second one below the 771km-depth layer. +# Setting this flag to .true., will move the second one below the 220km-depth layer for regional mesh cut-offs only.) +REGIONAL_MESH_ADD_2ND_DOUBLING = .false. + +#----------------------------------------------------------- +# +# Absorbing boundary conditions +# +#----------------------------------------------------------- + +# absorbing boundary conditions for a regional simulation +ABSORBING_CONDITIONS = .true. + +# run global simulation for a circular region and apply high attenuation for the rest of the model +# this creates an absorbing boundary with less reflection than stacey at a cost of 6x the computational cost +# NCHUNKS must be set to 6 to enable this flag +ABSORB_USING_GLOBAL_SPONGE = .false. + +# location and size of the region with no sponge (the region to run simulation) +SPONGE_LATITUDE_IN_DEGREES = 40.d0 +SPONGE_LONGITUDE_IN_DEGREES = 25.d0 +SPONGE_RADIUS_IN_DEGREES = 25.d0 + +#----------------------------------------------------------- +# +# undoing attenuation for sensitivity kernel calculations +# +#----------------------------------------------------------- + +# to undo attenuation for sensitivity kernel calculations or forward runs with SAVE_FORWARD +# use one (and only one) of the two flags below. UNDO_ATTENUATION is much better (it is exact) +# but requires a significant amount of disk space for temporary storage. +PARTIAL_PHYS_DISPERSION_ONLY = .false. +UNDO_ATTENUATION = .false. + +## undo attenuation memory +# How much memory (in GB) is installed on your machine per CPU core +# (only used for UNDO_ATTENUATION, can be ignored otherwise) +# Beware, this value MUST be given per core, i.e. per MPI thread, i.e. per MPI rank, NOT per node. +# This value is for instance: +# - 4 GB on Tiger at Princeton +# - 4 GB on TGCC Curie in Paris +# - 4 GB on Titan at ORNL when using CPUs only (no GPUs); start your run with "aprun -n$NPROC -N8 -S4 -j1" +# - 2 GB on the machine used by Christina Morency +# - 2 GB on the TACC machine used by Min Chen +# - 1.5 GB on the GPU cluster in Marseille +# When running on GPU machines, it is simpler to set PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0 +# and then set MEMORY_INSTALLED_PER_CORE_IN_GB to the amount of memory that you estimate is free (rather than installed) +# on the host of the GPU card while running your GPU job. +# For GPU runs on Titan at ORNL, use PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0 and MEMORY_INSTALLED_PER_CORE_IN_GB = 25.d0 +# and run your job with "aprun -n$NPROC -N1 -S1 -j1" +# (each host has 32 GB on Titan, each GPU has 6 GB, thus even if all the GPU arrays are duplicated on the host +# this leaves 32 - 6 = 26 GB free on the host; leaving 1 GB for the Linux system, we can safely use 100% of 25 GB) +MEMORY_INSTALLED_PER_CORE_IN_GB = 4.d0 +# What percentage of this total do you allow us to use for arrays to undo attenuation, keeping in mind that you +# need to leave some memory available for the GNU/Linux system to run +# (a typical value is 85%; any value below is fine but the code will then save a lot of data to disk; +# values above, say 90% or 92%, can be OK on some systems but can make the adjoint code run out of memory +# on other systems, depending on how much memory per node the GNU/Linux system needs for itself; thus you can try +# a higher value and if the adjoint crashes then try again with a lower value) +PERCENT_OF_MEM_TO_USE_PER_CORE = 85.d0 + +## exact mass matrices for rotation +# three mass matrices instead of one are needed to handle rotation very accurately; +# otherwise rotation is handled slightly less accurately (but still reasonably well); +# set to .true. if you are interested in precise effects related to rotation; +# set to .false. if you are solving very large inverse problems at high frequency and also undoing attenuation exactly +# +# using the UNDO_ATTENUATION flag above, in which case saving as much memory as possible can be a good idea. +# You can also safely set it to .false. if you are not in a period range in which rotation matters, +# e.g. if you are targetting very short-period body waves. if in doubt, set to .true. +# +# You can safeely set it to .true. if you have ABSORBING_CONDITIONS above, because in that case the code +# will use three mass matrices anyway and thus there is no additional memory cost. +# this flag is of course unused if ROTATION above is set to .false. +EXACT_MASS_MATRIX_FOR_ROTATION = .true. + +#----------------------------------------------------------- +# +# LDDRK time scheme +# +#----------------------------------------------------------- + +# this for LDDRK high-order time scheme instead of Newmark +USE_LDDRK = .false. + +# the maximum CFL of LDDRK is significantly higher than that of the Newmark scheme, +# in a ratio that is theoretically 1.327 / 0.697 = 1.15 / 0.604 = 1.903 for a solid with Poisson's ratio = 0.25 +# and for a fluid (see the manual of the 2D code, SPECFEM2D, Tables 4.1 and 4.2, and that ratio does not +# depend on whether we are in 2D or in 3D). However in practice a ratio of about 1.5 to 1.7 is often safer +# (for instance for models with a large range of Poisson's ratio values). +# Since the code computes the time step using the Newmark scheme, for LDDRK we will simply +# multiply that time step by this ratio when LDDRK is on and when flag INCREASE_CFL_FOR_LDDRK is true. +INCREASE_CFL_FOR_LDDRK = .true. +RATIO_BY_WHICH_TO_INCREASE_IT = 1.5d0 + +#----------------------------------------------------------- +# +# Visualization +# +#----------------------------------------------------------- + +# save AVS or OpenDX movies +#MOVIE_COARSE saves movie only at corners of elements (SURFACE OR VOLUME) +#MOVIE_COARSE does not work with create_movie_AVS_DX +MOVIE_SURFACE = .false. +MOVIE_VOLUME = .false. +MOVIE_COARSE = .true. +NTSTEP_BETWEEN_FRAMES = 50 +HDUR_MOVIE = 0.d0 + +# save movie in volume. Will save element if center of element is in prescribed volume +# top/bottom: depth in KM, use MOVIE_TOP = -100 to make sure the surface is stored. +# west/east: longitude, degrees East [-180/180] top/bottom: latitute, degrees North [-90/90] +# start/stop: frames will be stored at MOVIE_START + i*NSTEP_BETWEEN_FRAMES, where i=(0,1,2..) and iNSTEP_BETWEEN_FRAMES <= MOVIE_STOP +# movie_volume_type: 1=strain, 2=time integral of strain, 3=\mu*time integral of strain +# type 4 saves the trace and deviatoric stress in the whole volume, 5=displacement, 6=velocity +MOVIE_VOLUME_TYPE = 2 +MOVIE_TOP_KM = -100.0 +MOVIE_BOTTOM_KM = 1000.0 +MOVIE_WEST_DEG = -90.0 +MOVIE_EAST_DEG = 90.0 +MOVIE_NORTH_DEG = 90.0 +MOVIE_SOUTH_DEG = -90.0 +MOVIE_START = 0 +MOVIE_STOP = 40000 + +# save mesh files to check the mesh +SAVE_MESH_FILES = .false. + +# restart files (number of runs can be 1 or higher, choose 1 for no restart files) +NUMBER_OF_RUNS = 1 +NUMBER_OF_THIS_RUN = 1 + +# path to store the local database files on each node +LOCAL_PATH = ./DATABASES_MPI +# temporary wavefield/kernel/movie files +LOCAL_TMP_PATH = ./DATABASES_MPI + +# interval at which we output time step info and max of norm of displacement +NTSTEP_BETWEEN_OUTPUT_INFO = 500 + +#----------------------------------------------------------- +# +# Sources +# +#----------------------------------------------------------- + +# use a (tilted) FORCESOLUTION force point source (or several) instead of a CMTSOLUTION moment-tensor source. +# This can be useful e.g. for asteroid simulations +# in which the source is a vertical force, normal force, tilted force, impact etc. +# If this flag is turned on, the FORCESOLUTION file must be edited by giving: +# - the corresponding time-shift parameter, +# - the half duration parameter of the source, +# - the coordinates of the source, +# - the source time function of the source, +# - the magnitude of the force source, +# - the components of a (non necessarily unitary) direction vector for the force source in the E/N/Z_UP basis. +# The direction vector is made unitary internally in the code and thus only its direction matters here; +# its norm is ignored and the norm of the force used is the factor force source times the source time function. +USE_FORCE_POINT_SOURCE = .true. + +# use monochromatic source time function for CMTSOLUTION moment-tensor source. +# half duration is interpreted as a PERIOD just to avoid changing CMTSOLUTION file format +# default is .false. which uses Heaviside function +USE_MONOCHROMATIC_CMT_SOURCE = .false. + +# uses a sin-squared STF useful for PEGS calculations +# see https://doi.org/10.1016/j.epsl.2020.116150 for definition +USE_SINSQ_STF = .false. + +# print source time function +PRINT_SOURCE_TIME_FUNCTION = .false. + +#----------------------------------------------------------- +# +# Seismograms +# +#----------------------------------------------------------- + +# interval in time steps for temporary writing of seismograms +NTSTEP_BETWEEN_OUTPUT_SEISMOS = 5000000 + +# set to n to reduce the sampling rate of output seismograms by a factor of n +# defaults to 1, which means no down-sampling +NTSTEP_BETWEEN_OUTPUT_SAMPLE = 1 + +# option to save strain seismograms +# this option is useful for strain Green's tensor +# this feature is currently under development +SAVE_SEISMOGRAMS_STRAIN = .false. + +# save seismograms also when running the adjoint runs for an inverse problem +# (usually they are unused and not very meaningful, leave this off in almost all cases) +SAVE_SEISMOGRAMS_IN_ADJOINT_RUN = .false. + +# output format for the seismograms (one can use either or all of the three formats) +OUTPUT_SEISMOS_ASCII_TEXT = .false. +OUTPUT_SEISMOS_SAC_ALPHANUM = .false. +OUTPUT_SEISMOS_SAC_BINARY = .true. +OUTPUT_SEISMOS_ASDF = .false. +OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. + +# rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame +ROTATE_SEISMOGRAMS_RT = .false. + +# decide if main process writes all the seismograms or if all processes do it in parallel +WRITE_SEISMOGRAMS_BY_MAIN = .false. + +# save all seismograms in one large combined file instead of one file per seismogram +# to avoid overloading shared non-local file systems such as LUSTRE or GPFS for instance +SAVE_ALL_SEISMOS_IN_ONE_FILE = .false. +USE_BINARY_FOR_LARGE_FILE = .false. + +# flag to impose receivers at the surface or allow them to be buried +RECEIVERS_CAN_BE_BURIED = .true. + +#----------------------------------------------------------- +# +# Adjoint kernel outputs +# +#----------------------------------------------------------- + +# interval in time steps for reading adjoint traces +# 0 = read the whole adjoint sources at start time +NTSTEP_BETWEEN_READ_ADJSRC = 0 + +# use ASDF format for reading the adjoint sources +READ_ADJSRC_ASDF = .false. + +# this parameter must be set to .true. to compute anisotropic kernels +# in crust and mantle (related to the 21 Cij in geographical coordinates) +# default is .false. to compute isotropic kernels (related to alpha and beta) +ANISOTROPIC_KL = .false. + +# output only transverse isotropic kernels (alpha_v,alpha_h,beta_v,beta_h,eta,rho) +# rather than fully anisotropic kernels when ANISOTROPIC_KL above is set to .true. +# means to save radial anisotropic kernels, i.e., sensitivity kernels for beta_v, beta_h, etc. +SAVE_TRANSVERSE_KL_ONLY = .false. + +# output only the kernels used for the current azimuthally anisotropic inversions of surface waves, +# i.e., bulk_c, bulk_betav, bulk_betah, eta, Gc_prime, Gs_prime and rho +# (Gc' & Gs' which are the normalized Gc & Gs kernels by isotropic \rho\beta of the 1D reference model) +SAVE_AZIMUTHAL_ANISO_KL_ONLY = .false. + +# output approximate Hessian in crust mantle region. +# means to save the preconditioning for gradients, they are cross correlations between forward and adjoint accelerations. +APPROXIMATE_HESS_KL = .false. + +# forces transverse isotropy for all mantle elements +# (default is to use transverse isotropy only between crust and 220) +# means we allow radial anisotropy throughout the whole crust/mantle region +USE_FULL_TISO_MANTLE = .false. + +# output kernel mask to zero out source region +# to remove large values near the sources in the sensitivity kernels +SAVE_SOURCE_MASK = .false. + +# output kernels on a regular grid instead of on the GLL mesh points (a bit expensive) +SAVE_REGULAR_KL = .false. + +# compute steady state kernels for source encoding +STEADY_STATE_KERNEL = .false. +STEADY_STATE_LENGTH_IN_MINUTES = 0.d0 + +#----------------------------------------------------------- + +# Dimitri Komatitsch, July 2014, CNRS Marseille, France: +# added the ability to run several calculations (several earthquakes) +# in an embarrassingly-parallel fashion from within the same run; +# this can be useful when using a very large supercomputer to compute +# many earthquakes in a catalog, in which case it can be better from +# a batch job submission point of view to start fewer and much larger jobs, +# each of them computing several earthquakes in parallel. +# To turn that option on, set parameter NUMBER_OF_SIMULTANEOUS_RUNS to a value greater than 1. +# To implement that, we create NUMBER_OF_SIMULTANEOUS_RUNS MPI sub-communicators, +# each of them being labeled "my_local_mpi_comm_world", and we use them +# in all the routines in "src/shared/parallel.f90", except in MPI_ABORT() because in that case +# we need to kill the entire run. +# When that option is on, of course the number of processor cores used to start +# the code in the batch system must be a multiple of NUMBER_OF_SIMULTANEOUS_RUNS, +# all the individual runs must use the same number of processor cores, +# which as usual is NPROC in the Par_file, +# and thus the total number of processor cores to request from the batch system +# should be NUMBER_OF_SIMULTANEOUS_RUNS * NPROC. +# All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on +# (with exactly four digits). +# +# Imagine you have 10 independent calculations to do, each of them on 100 cores; you have three options: +# +# 1/ submit 10 jobs to the batch system +# +# 2/ submit a single job on 1000 cores to the batch, and in that script create a sub-array of jobs to start 10 jobs, +# each running on 100 cores (see e.g. http://www.schedmd.com/slurmdocs/job_array.html ) +# +# 3/ submit a single job on 1000 cores to the batch, start SPECFEM3D on 1000 cores, create 10 sub-communicators, +# cd into one of 10 subdirectories (called e.g. run0001, run0002,... run0010) depending on the sub-communicator +# your MPI rank belongs to, and run normally on 100 cores using that sub-communicator. +# +# The option below implements 3/. +# +NUMBER_OF_SIMULTANEOUS_RUNS = 1 + +# if we perform simultaneous runs in parallel, if only the source and receivers vary between these runs +# but not the mesh nor the model (velocity and density) then we can also read the mesh and model files +# from a single run in the beginning and broadcast them to all the others; for a large number of simultaneous +# runs for instance when solving inverse problems iteratively this can DRASTICALLY reduce I/Os to disk in the solver +# (by a factor equal to NUMBER_OF_SIMULTANEOUS_RUNS), and reducing I/Os is crucial in the case of huge runs. +# Thus, always set this option to .true. if the mesh and the model are the same for all simultaneous runs. +# In that case there is no need to duplicate the mesh and model file database (the content of the DATABASES_MPI +# directories) in each of the run0001, run0002,... directories, it is sufficient to have one in run0001 +# and the code will broadcast it to the others) +BROADCAST_SAME_MESH_AND_MODEL = .false. + +#----------------------------------------------------------- + +# set to true to use GPUs +GPU_MODE = .false. +# Only used if GPU_MODE = .true. : +GPU_RUNTIME = 1 +# 2 (OpenCL), 1 (Cuda) ou 0 (Compile-time -- does not work if configured with --with-cuda *AND* --with-opencl) +GPU_PLATFORM = NVIDIA +GPU_DEVICE = Tesla + +# set to true to use the ADIOS library for I/Os +ADIOS_ENABLED = .false. +ADIOS_FOR_FORWARD_ARRAYS = .true. +ADIOS_FOR_MPI_ARRAYS = .true. +ADIOS_FOR_ARRAYS_SOLVER = .true. +ADIOS_FOR_SOLVER_MESHFILES = .true. +ADIOS_FOR_AVS_DX = .true. +ADIOS_FOR_KERNELS = .true. +ADIOS_FOR_MODELS = .true. +ADIOS_FOR_UNDO_ATTENUATION = .true. + +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .true. + +# Green function database +GF_DATABASE_ENABLED = .true. +GF_DATABASE_PATH = /Users/lsawade/specfem3d_globe/EXAMPLES/green_function_database/regional/GFDB/ +GF_SUBSAMPLE_STEP = 4 +GF_BUFFER_SIZE = 100 +GF_NEIGHBOR_SHELLS = 1 +DT = 0.1 \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/STATIONS b/EXAMPLES/green_function_database/regional/db_base/DATA/STATIONS new file mode 100644 index 000000000..aa8dc064a --- /dev/null +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/STATIONS @@ -0,0 +1 @@ + SJG IU 18.1091 -66.1500 420.0 0.0 diff --git a/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION b/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION new file mode 100644 index 000000000..7c3036457 --- /dev/null +++ b/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION @@ -0,0 +1,13 @@ +PDE 1994 6 9 0 33 16.40 -13.8300 -67.5600 637.0 6.9 6.8 NORTHERN BOLIVIA +event name: 060994A +time shift: 29.0000 +half duration: 60.0000 +latitude: -13.8200 +longitude: -67.2500 +depth: 647.1000 +Mrr: -7.590000e+27 +Mtt: 7.750000e+27 +Mpp: -1.600000e+26 +Mrt: -2.503000e+28 +Mrp: 4.200000e+26 +Mtp: -2.480000e+27 diff --git a/EXAMPLES/green_function_database/regional/validation_data/FORCESOLUTION b/EXAMPLES/green_function_database/regional/validation_data/FORCESOLUTION new file mode 100644 index 000000000..ca391f62e --- /dev/null +++ b/EXAMPLES/green_function_database/regional/validation_data/FORCESOLUTION @@ -0,0 +1,11 @@ +FORCE 001 +time shift: 0.0000 +f0: 45.0000 +latitude: -5.8120 +longitude: -75.2700 +depth: 122.6000 +source time function: 0 +factor force source: 1.0d15 +comp dir vect source E: 1.0 +comp dir vect source N: 0.0 +comp dir vect source Z: 0.0 diff --git a/EXAMPLES/green_function_database/regional/validation_data/STATIONS b/EXAMPLES/green_function_database/regional/validation_data/STATIONS new file mode 100644 index 000000000..c0f5f2ca8 --- /dev/null +++ b/EXAMPLES/green_function_database/regional/validation_data/STATIONS @@ -0,0 +1,18 @@ + EFI II -51.6753 -58.0637 110.0 80.0 + HOPE II -54.2836 -36.4879 20.0 0.0 + JTS II 10.2908 -84.9525 340.0 0.0 + NNA II -11.9875 -76.8422 575.0 40.0 + RPN II -27.1267 -109.3344 110.0 0.0 + BOCO IU 4.5869 -74.0432 3137.0 23.0 + DWPF IU 28.1103 -81.4327 -132.0 162.0 + LCO IU -29.0110 -70.7004 2300.0 0.0 + LVC IU -22.6127 -68.9111 2930.0 30.0 + OTAV IU 0.2376 -78.4508 3495.0 15.0 + PAYG IU -0.6742 -90.2861 170.0 100.0 + PTGA IU -0.7308 -59.9666 141.0 96.0 + RCBR IU -5.8274 -35.9014 291.0 109.0 + SAML IU -8.9489 -63.1831 120.0 0.0 + SDV IU 8.8839 -70.6340 1588.0 32.0 + SJG IU 18.1091 -66.1500 420.0 0.0 + TEIG IU 20.2263 -88.2763 15.0 25.0 + TRQA IU -38.0568 -61.9787 439.0 101.0 diff --git a/src/specfem3D/green_function_metadata.F90 b/src/specfem3D/green_function_metadata.F90 index 3275e1df9..aa57a4ea3 100644 --- a/src/specfem3D/green_function_metadata.F90 +++ b/src/specfem3D/green_function_metadata.F90 @@ -240,6 +240,10 @@ subroutine gf_write_mesh_info() double precision :: attr_dp(1) integer :: attr_int(1) + ! validation + double precision :: existing_dt(1) + integer :: existing_nstep(1), existing_subsample(1) + if (myrank /= 0) return ! initialize HDF5 Fortran interface (reference-counted, safe to call multiple times) @@ -257,6 +261,48 @@ subroutine gf_write_mesh_info() call h5aopen_f(fid, 'scale_displ', attr_id, hdferr) if (hdferr == 0) then call h5aclose_f(attr_id, hdferr) + + ! validate that simulation parameters match the current run + call h5aopen_f(fid, 'dt', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_DOUBLE, existing_dt, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + if (abs(existing_dt(1) - DT) > 1.0d-12) then + call h5fclose_f(fid, hdferr) + write(IMAIN,*) 'ERROR: mesh_info.h5 has dt =', existing_dt(1), ' but current run has DT =', DT + write(IMAIN,*) 'Delete the existing mesh_info.h5 and restart.' + call flush_IMAIN() + call exit_MPI(myrank, 'mesh_info.h5 dt mismatch — delete and restart') + endif + endif + + call h5aopen_f(fid, 'nstep', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, existing_nstep, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + if (existing_nstep(1) /= NSTEP) then + call h5fclose_f(fid, hdferr) + write(IMAIN,*) 'ERROR: mesh_info.h5 has nstep =', existing_nstep(1), ' but current run has NSTEP =', NSTEP + write(IMAIN,*) 'Delete the existing mesh_info.h5 and restart.' + call flush_IMAIN() + call exit_MPI(myrank, 'mesh_info.h5 nstep mismatch — delete and restart') + endif + endif + + call h5aopen_f(fid, 'subsample_step', attr_id, hdferr) + if (hdferr == 0) then + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, existing_subsample, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + if (existing_subsample(1) /= GF_SUBSAMPLE_STEP) then + call h5fclose_f(fid, hdferr) + write(IMAIN,*) 'ERROR: mesh_info.h5 has subsample_step =', existing_subsample(1), & + ' but current run has GF_SUBSAMPLE_STEP =', GF_SUBSAMPLE_STEP + write(IMAIN,*) 'Delete the existing mesh_info.h5 and restart.' + call flush_IMAIN() + call exit_MPI(myrank, 'mesh_info.h5 subsample_step mismatch — delete and restart') + endif + endif + call h5fclose_f(fid, hdferr) write(IMAIN,*) 'Green function database: mesh_info.h5 already exists, skipping' write(IMAIN,*) From c8f25d61e6ab985596e185fb843e05eb844873e4 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 14:24:10 -0400 Subject: [PATCH 12/20] Updated the python validation scripts as well as the workflow to include the validation script plot. --- .../regional/.gitignore | 9 +- .../regional/Snakefile | 138 +- .../regional/validation_data/CMTSOLUTION | 6 +- utils/green_function/gf_cross_validate.py | 1242 ++++++++++------- utils/green_function/xvalidate.py | 159 +++ 5 files changed, 1065 insertions(+), 489 deletions(-) create mode 100644 utils/green_function/xvalidate.py diff --git a/EXAMPLES/green_function_database/regional/.gitignore b/EXAMPLES/green_function_database/regional/.gitignore index 143bf66b2..38b8907fa 100644 --- a/EXAMPLES/green_function_database/regional/.gitignore +++ b/EXAMPLES/green_function_database/regional/.gitignore @@ -1,4 +1,11 @@ .snakemake +db_base/DATA/crust2.0 +db_base/DATA/s362ani +db_base/DATA/topo_bathy db_base/DATABASES_MPI db_base/OUTPUT_FILES -db_base/mesher.flag \ No newline at end of file +db_base/mesher.flag +forward +forward_cmt +simulations +validation_output \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/Snakefile b/EXAMPLES/green_function_database/regional/Snakefile index 74f825228..ed0e9b0fc 100644 --- a/EXAMPLES/green_function_database/regional/Snakefile +++ b/EXAMPLES/green_function_database/regional/Snakefile @@ -54,6 +54,7 @@ SIMDIR = config.get("SIMDIR", "simulations") CREATE_VALIDATION = config.get("CREATE_VALIDATION", True) VALIDATION_DATA = config.get("VALIDATION_DATA", "validation_data") VALIDATION_DIR = config.get("VALIDATION_DIR", "forward") +VALIDATION_CMT_DIR = config.get("VALIDATION_CMT_DIR", "forward_cmt") # ─── GF database output path ───────────────────────────────────────────────── @@ -132,9 +133,10 @@ def all_inputs(wildcards): station=STATION_IDS) # Manifest built inputs.append(os.path.join(GF_DATABASE_PATH, "centroids.bin")) - # Optionally run validation forward simulation + # Optionally run validation forward simulations and cross-validation plots if CREATE_VALIDATION: - inputs.append(os.path.join(VALIDATION_DIR, "done.flag")) + inputs.append(os.path.join("validation_output", "xvalidate_force.svg")) + inputs.append(os.path.join("validation_output", "xvalidate_cmt.svg")) return inputs @@ -391,6 +393,13 @@ rule setup_validation_forward: parfile_content, flags=re.MULTILINE, ) + # Ensure force source is enabled for force validation + parfile_content = re.sub( + r"^(USE_FORCE_POINT_SOURCE\s*=\s*)\.false\.", + r"\1.true.", + parfile_content, + flags=re.MULTILINE, + ) with open(parfile_dst, "w") as f: f.write(parfile_content) @@ -438,6 +447,118 @@ rule run_validation_forward: f.write("validation forward complete\n") +rule setup_validation_cmt: + """Set up a CMT forward simulation directory for validation.""" + input: + # Only run after the GF database is built + os.path.join(GF_DATABASE_PATH, "centroids.bin"), + output: + flag=os.path.join(VALIDATION_CMT_DIR, "setup.flag"), + params: + basedir=BASEDIR, + validation_data=VALIDATION_DATA, + validation_cmt_dir=VALIDATION_CMT_DIR, + run: + import re + + vdir = params.validation_cmt_dir + + # Create directories + os.makedirs(os.path.join(vdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(vdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ from base directory + bin_link = os.path.join(vdir, "bin") + db_link = os.path.join(vdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file from base directory, disable GF and set CMT source + parfile_src = os.path.join(params.basedir, "DATA", "Par_file") + parfile_dst = os.path.join(vdir, "DATA", "Par_file") + with open(parfile_src, "r") as f: + parfile_content = f.read() + parfile_content = re.sub( + r"^(\s*GF_DATABASE_ENABLED\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + # Disable force point source so SPECFEM uses CMTSOLUTION + parfile_content = re.sub( + r"^(USE_FORCE_POINT_SOURCE\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + with open(parfile_dst, "w") as f: + f.write(parfile_content) + + # Copy STATIONS from validation_data + shutil.copy2( + os.path.join(params.validation_data, "STATIONS"), + os.path.join(vdir, "DATA", "STATIONS"), + ) + + # Copy CMTSOLUTION (required for CMT simulation) + cmt_src = os.path.join(params.validation_data, "CMTSOLUTION") + if os.path.exists(cmt_src): + shutil.copy2(cmt_src, os.path.join(vdir, "DATA", "CMTSOLUTION")) + else: + raise FileNotFoundError(f"CMTSOLUTION not found at {cmt_src}") + + # Touch flag + with open(output.flag, "w") as f: + f.write("validation CMT setup complete\n") + + +rule run_validation_cmt: + """Run the CMT validation forward simulation.""" + input: + setup=os.path.join(VALIDATION_CMT_DIR, "setup.flag"), + output: + flag=os.path.join(VALIDATION_CMT_DIR, "done.flag"), + params: + validation_cmt_dir=VALIDATION_CMT_DIR, + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + vdir = params.validation_cmt_dir + shell( + "cd {vdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver.txt 2>&1" + ) + with open(output.flag, "w") as f: + f.write("validation CMT forward complete\n") + + +rule run_cross_validation: + """Run cross-validation comparing GF database against forward simulations.""" + input: + force_done=os.path.join(VALIDATION_DIR, "done.flag"), + cmt_done=os.path.join(VALIDATION_CMT_DIR, "done.flag"), + output: + force_svg=os.path.join("validation_output", "xvalidate_force.svg"), + cmt_svg=os.path.join("validation_output", "xvalidate_cmt.svg"), + params: + script=os.path.join( + os.path.abspath(SPECFEM_DIR), + "utils", "green_function", "xvalidate.py", + ), + shell: + """ + python {params.script} . + """ + + rule clean_base: """Clean base directory links and mesher output.""" shell: @@ -455,6 +576,16 @@ rule clean_validation: shell: "rm -rf {VALIDATION_DIR}/*" +rule clean_validation_cmt: + """Clean CMT validation forward simulation directory.""" + shell: + "rm -rf {VALIDATION_CMT_DIR}/*" + +rule clean_cross_validation: + """Clean cross-validation output files.""" + shell: + "rm -rf validation_output/" + rule clean_database: """Clean GF database output files.""" shell: @@ -466,4 +597,5 @@ rule clean_all: "rm -rf {BASEDIR}/bin {BASEDIR}/DATABASES_MPI {BASEDIR}/OUTPUT_FILES " "{BASEDIR}/DATA/topo_bathy {BASEDIR}/DATA/s362ani {BASEDIR}/DATA/crust2.0 " "{BASEDIR}/mesher.flag " - "{SIMDIR}/* {GF_DATABASE_PATH}/* {VALIDATION_DIR}/*" \ No newline at end of file + "{SIMDIR}/* {GF_DATABASE_PATH}/* {VALIDATION_DIR}/* {VALIDATION_CMT_DIR}/* " + "validation_output/*" \ No newline at end of file diff --git a/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION b/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION index 7c3036457..74c3eed61 100644 --- a/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION +++ b/EXAMPLES/green_function_database/regional/validation_data/CMTSOLUTION @@ -2,9 +2,9 @@ PDE 1994 6 9 0 33 16.40 -13.8300 -67.5600 637.0 6.9 6.8 NORTHERN BOLIVIA event name: 060994A time shift: 29.0000 half duration: 60.0000 -latitude: -13.8200 -longitude: -67.2500 -depth: 647.1000 +latitude: -5.8120 +longitude: -75.2700 +depth: 122.6000 Mrr: -7.590000e+27 Mtt: 7.750000e+27 Mpp: -1.600000e+26 diff --git a/utils/green_function/gf_cross_validate.py b/utils/green_function/gf_cross_validate.py index 1ea644feb..9a05c2642 100644 --- a/utils/green_function/gf_cross_validate.py +++ b/utils/green_function/gf_cross_validate.py @@ -2,33 +2,37 @@ """ Cross-validate Green function database against a forward simulation. -Reads the GF displacement from the HDF5 database, interpolates to the -source location using Lagrange interpolation on GLL nodes, applies -reciprocity rotation, and compares against the forward seismogram. - -The forward simulation must use USE_FORCE_POINT_SOURCE = .true. with the -source placed at a location covered by the GF database (i.e., one of -the GF_LOCATIONS entries). +Supports two source types: + - Force source (--force): Reads GF displacement, interpolates to source, + applies reciprocity with force direction. The forward simulation must + use USE_FORCE_POINT_SOURCE = .true. + - Moment tensor / CMT (default): Computes strain from GF displacement + gradients, rotates the moment tensor to Cartesian coordinates, contracts + strain with MT, and time-integrates to convert from Gaussian to Heaviside + STF. The forward simulation must use USE_FORCE_POINT_SOURCE = .false. + +In both cases, the source must be placed at a location covered by the GF +database (i.e., one of the GF_LOCATIONS entries). Usage: python gf_cross_validate.py [options] -Example: - python gf_cross_validate.py \ - DB_TESTING/db_gen/OUTPUT_FILES/gf_database \ - DB_TESTING/forward/OUTPUT_FILES \ - --forward-solver-output DB_TESTING/forward/OUTPUT_FILES/output_solver.txt \ - --station IU.SJG +Examples: + # CMT (default): + python gf_cross_validate.py GFDB/ forward_cmt/OUTPUT_FILES --station IU.SJG + + # Force source: + python gf_cross_validate.py GFDB/ forward/OUTPUT_FILES --force --station IU.SJG """ import argparse -import os import re import sys from pathlib import Path from math import sin, cos, radians, sqrt, pi import numpy as np +from scipy.signal import fftconvolve try: import h5py @@ -49,15 +53,31 @@ sys.exit(1) +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + # GLL points for NGLL=5 (Gauss-Lobatto-Legendre quadrature nodes on [-1,1]) GLL5 = np.array([-1.0, -0.6546536707079771, 0.0, 0.6546536707079771, 1.0]) +# SPECFEM3D non-dimensionalization constants (matching setup/constants.h.in) +R_PLANET = 6371000.0 # Earth radius in meters +RHOAV = 5514.3 # Average density in kg/m^3 +GRAV = 6.67384e-11 # Gravitational constant in m^3 kg^-1 s^-2 -def lagrange_basis(xi, nodes): - """Compute Lagrange basis function values at point xi for given nodes. +# Moment tensor scaling: M_nondim = M_dyne_cm / SCALE_M +# From get_cmt.f90: scaleM = 1.d7 * RHOAV * (R_PLANET**5) * PI * GRAV * RHOAV +SCALE_M = 1.0e7 * RHOAV * (R_PLANET ** 5) * pi * GRAV * RHOAV - Returns array of length len(nodes) with L_i(xi) for each i. - """ +SOURCE_DECAY_MIMIC_TRIANGLE = 1.628 + + +# --------------------------------------------------------------------------- +# GLL interpolation and derivatives +# --------------------------------------------------------------------------- + +def lagrange_basis(xi, nodes): + """Compute Lagrange basis function values at point xi for given nodes.""" n = len(nodes) L = np.ones(n) for i in range(n): @@ -67,6 +87,21 @@ def lagrange_basis(xi, nodes): return L +def lagrange_deriv(xi, nodes): + """Compute derivative of Lagrange basis functions at point xi.""" + n = len(nodes) + dL = np.zeros(n) + for i in range(n): + for j in range(n): + if j != i: + prod = 1.0 / (nodes[i] - nodes[j]) + for k in range(n): + if k != i and k != j: + prod *= (xi - nodes[k]) / (nodes[i] - nodes[k]) + dL[i] += prod + return dL + + def interpolate_gll(data, xi, eta, gamma): """Interpolate data on a 5x5x5 GLL element to point (xi, eta, gamma). @@ -83,90 +118,15 @@ def interpolate_gll(data, xi, eta, gamma): result : ndarray, shape (nt, 3, 3) Interpolated values. """ - Li = lagrange_basis(xi, GLL5) # basis for i-direction (dim 3) - Lj = lagrange_basis(eta, GLL5) # basis for j-direction (dim 2) - Lk = lagrange_basis(gamma, GLL5) # basis for k-direction (dim 1) - - # Contract over GLL dimensions (dims 1, 2, 3) - # data[t, k, j, i, d, f] -> sum over i,j,k with Li, Lj, Lk - result = np.einsum("tkjidf,i,j,k->tdf", data, Li, Lj, Lk) - return result - - -def parse_solver_output(filepath): - """Parse source info from specfem3D output_solver.txt. - - Returns dict with: xi, eta, gamma, x, y, z, nu1, nu2, nu3, lat, lon, depth - """ - text = Path(filepath).read_text() + Li = lagrange_basis(xi, GLL5) + Lj = lagrange_basis(eta, GLL5) + Lk = lagrange_basis(gamma, GLL5) + return np.einsum("tkjidf,i,j,k->tdf", data, Li, Lj, Lk) - info = {} - - # Parse xi, eta, gamma - m = re.search( - r"at xi,eta,gamma coordinates\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", - text, - ) - if m: - info["xi"] = float(m.group(1)) - info["eta"] = float(m.group(2)) - info["gamma"] = float(m.group(3)) - - # Parse (x, y, z) - m = re.search( - r"at \(x,y,z\)\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text - ) - if m: - info["x"] = float(m.group(1)) - info["y"] = float(m.group(2)) - info["z"] = float(m.group(3)) - - # Parse nu1, nu2, nu3 (rotation matrix rows: North, East, Vertical) - for name in ["nu1", "nu2", "nu3"]: - m = re.search( - rf"{name}\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text - ) - if m: - info[name] = np.array( - [float(m.group(1)), float(m.group(2)), float(m.group(3))] - ) - - # Parse original source position - m = re.search(r"latitude:\s*([-\dE.+]+)", text) - if m: - info["lat"] = float(m.group(1)) - m = re.search(r"longitude:\s*([-\dE.+]+)", text) - if m: - info["lon"] = float(m.group(1)) - - # Parse force direction - m = re.search(r"East direction:\s*([-\dE.+]+)", text) - if m: - info["force_E"] = float(m.group(1)) - m = re.search(r"North direction:\s*([-\dE.+]+)", text) - if m: - info["force_N"] = float(m.group(1)) - m = re.search(r"Vertical direction:\s*([-\dE.+]+)", text) - if m: - info["force_Z"] = float(m.group(1)) - - return info - - -def lagrange_deriv(xi, nodes): - """Compute derivative of Lagrange basis functions at point xi.""" - n = len(nodes) - dL = np.zeros(n) - for i in range(n): - for j in range(n): - if j != i: - prod = 1.0 / (nodes[i] - nodes[j]) - for k in range(n): - if k != i and k != j: - prod *= (xi - nodes[k]) / (nodes[i] - nodes[k]) - dL[i] += prod - return dL +# --------------------------------------------------------------------------- +# Element location (Newton-Raphson) +# --------------------------------------------------------------------------- def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): """Find reference coordinates (xi, eta, gamma) of a point in a spectral element. @@ -179,10 +139,6 @@ def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): GLL node coordinates of the element. target : ndarray, shape (3,) Target point in Cartesian coordinates. - maxiter : int - Maximum Newton iterations. - tol : float - Convergence tolerance on residual norm. Returns ------- @@ -198,7 +154,6 @@ def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): Lj = lagrange_basis(eta, GLL5) Lk = lagrange_basis(gamma, GLL5) - # Current mapped position current = np.array([ np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, Lj, Lk) for d in range(3) @@ -208,7 +163,6 @@ def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): if np.linalg.norm(residual) < tol: return xi, eta, gamma, True - # Jacobian via derivatives of Lagrange basis dLi = lagrange_deriv(xi, GLL5) dLj = lagrange_deriv(eta, GLL5) dLk = lagrange_deriv(gamma, GLL5) @@ -224,7 +178,6 @@ def locate_point_in_element(xyz_elem, target, maxiter=20, tol=1e-10): eta += dxi[1] gamma += dxi[2] - # Clamp to avoid divergence xi = np.clip(xi, -1.5, 1.5) eta = np.clip(eta, -1.5, 1.5) gamma = np.clip(gamma, -1.5, 1.5) @@ -263,10 +216,9 @@ def find_containing_element(gf_db_path, source_xyz): ) candidates.append((dist, elem_dir.name)) - # Sort by centroid distance and check nearest candidates candidates.sort() - for dist, morton in candidates[:20]: # check top 20 nearest + for dist, morton in candidates[:20]: coord_file = elements_dir / morton / "coordinates.h5" with h5py.File(coord_file, "r") as f: xyz_elem = f["xyz"][:] @@ -276,32 +228,26 @@ def find_containing_element(gf_db_path, source_xyz): if converged and all(abs(v) <= 1.01 for v in (xi, eta, gamma)): return morton, xi, eta, gamma, dist - # Fallback: return closest centroid (may not contain the point) morton = candidates[0][1] dist = candidates[0][0] print(f" WARNING: no element contains the source, using closest centroid {morton}") return morton, 0.0, 0.0, 0.0, dist -def butterworth_highpass_sos(order, fc, fs): - """Compute Butterworth highpass SOS coefficients. +# --------------------------------------------------------------------------- +# Butterworth filtering (matching Fortran implementation) +# --------------------------------------------------------------------------- - Returns ndarray of shape (order//2, 6) with [b0, b1, b2, a0=1, a1, a2] per section. - """ +def butterworth_highpass_sos(order, fc, fs): + """Compute Butterworth highpass SOS coefficients.""" nsections = order // 2 sos = np.zeros((nsections, 6)) - - # Pre-warp cutoff frequency wc = 2.0 * fs * np.tan(pi * fc / fs) for k in range(1, nsections + 1): - # Analog Butterworth pole angle angle = pi * (2 * k + order - 1) / (2 * order) pole_real = wc * cos(angle) - # Bilinear transform: highpass has s^2 numerator in analog domain - # H_hp(s) = s^2 / (s^2 - 2*Re(p)*s + |p|^2) for each conjugate pair - # After bilinear transform z = (1 + s/(2fs)) / (1 - s/(2fs)): b0 = 4.0 * fs * fs b1 = -8.0 * fs * fs b2 = 4.0 * fs * fs @@ -310,31 +256,22 @@ def butterworth_highpass_sos(order, fc, fs): a1 = 2.0 * wc * wc - 8.0 * fs * fs a2 = 4.0 * fs * fs + 4.0 * fs * pole_real + wc * wc - # Normalize sos[k - 1] = [b0 / a0, b1 / a0, b2 / a0, 1.0, a1 / a0, a2 / a0] return sos def butterworth_sos(order, fc, fs): - """Compute Butterworth lowpass SOS coefficients (matching Fortran implementation). - - Returns ndarray of shape (order//2, 6) with [b0, b1, b2, a0=1, a1, a2] per section. - """ + """Compute Butterworth lowpass SOS coefficients (matching Fortran implementation).""" nsections = order // 2 sos = np.zeros((nsections, 6)) - - # Pre-warp cutoff frequency wc = 2.0 * fs * np.tan(pi * fc / fs) wc2 = wc * wc for k in range(1, nsections + 1): - # Analog Butterworth pole angle angle = pi * (2 * k + order - 1) / (2 * order) pole_real = wc * cos(angle) - pole_imag = wc * sin(angle) - # Bilinear transform coefficients b0 = wc2 b1 = 2.0 * wc2 b2 = wc2 @@ -343,7 +280,6 @@ def butterworth_sos(order, fc, fs): a1 = 2.0 * wc2 - 8.0 * fs * fs a2 = 4.0 * fs * fs + 4.0 * fs * pole_real + wc2 - # Normalize sos[k - 1] = [b0 / a0, b1 / a0, b2 / a0, 1.0, a1 / a0, a2 / a0] return sos @@ -363,17 +299,11 @@ def sos_filter_forward(x, b0, b1, b2, a1, a2): def sosfiltfilt(x, sos): - """Zero-phase SOS filter matching Fortran implementation. - - For each section: forward pass, then reverse + forward + reverse (backward pass). - This matches the Fortran gf_sosfiltfilt which processes one section at a time. - """ + """Zero-phase SOS filter matching Fortran implementation.""" y = x.copy() for isec in range(sos.shape[0]): b0, b1, b2, _, a1, a2 = sos[isec] - # Forward pass y = sos_filter_forward(y, b0, b1, b2, a1, a2) - # Backward pass (reverse, filter, reverse) y = y[::-1].copy() y = sos_filter_forward(y, b0, b1, b2, a1, a2) y = y[::-1].copy() @@ -381,421 +311,769 @@ def sosfiltfilt(x, sos): def filter_with_taper(x, sos, taper_fraction=0.05, pad_factor=1.0): - """Taper signal edges to zero, zero-pad, filter, and trim. - - The taper brings the signal smoothly to zero at both ends so the - transition to zero-padding is seamless. The padding then absorbs - IIR filter edge transients. After filtering, the result is trimmed - back to the original length. - """ + """Taper signal edges to zero, zero-pad, filter, and trim.""" n = len(x) n_taper = max(1, int(taper_fraction * n)) n_pad = int(pad_factor * n) - # Cosine taper to bring edges to zero taper = np.ones(n) taper[:n_taper] = 0.5 * (1.0 - np.cos(np.linspace(0, pi, n_taper))) taper[-n_taper:] = 0.5 * (1.0 - np.cos(np.linspace(pi, 0, n_taper))) tapered = x * taper - # Zero-pad on both sides padded = np.concatenate([np.zeros(n_pad), tapered, np.zeros(n_pad)]) - - # Filter and trim back to original length filtered = sosfiltfilt(padded, sos) return filtered[n_pad:n_pad + n] -def main(): - parser = argparse.ArgumentParser( - description="Cross-validate GF database against forward simulation" - ) - parser.add_argument( - "gf_db_path", type=Path, help="Path to GF database directory" - ) - parser.add_argument( - "forward_output", type=Path, help="Path to forward OUTPUT_FILES directory" - ) - parser.add_argument( - "--forward-solver-output", - type=Path, - default=None, - help="Path to forward output_solver.txt (default: /output_solver.txt)", - ) - parser.add_argument( - "--station", - type=str, - default="IU.SJG", - help="Station name as NET.STA (default: IU.SJG)", - ) - parser.add_argument( - "--gf-solver-output", - type=Path, - default=None, - help="Path to GF output_solver.txt (to get t0_gf; default: auto-detect)", - ) - parser.add_argument( - "--output", - type=str, - default=None, - help="Output plot path (default: gf_cross_validation.png next to forward output)", - ) - parser.add_argument( - "--highpass-period", - type=float, - default=None, - help="Apply zero-phase Butterworth highpass filter at this period (in seconds). " - "Removes energy at periods longer than this value.", +# --------------------------------------------------------------------------- +# Source file parsers +# --------------------------------------------------------------------------- + +def parse_solver_output(filepath): + """Parse source info from specfem3D output_solver.txt. + + Returns dict with: xi, eta, gamma, x, y, z, nu1, nu2, nu3, lat, lon, depth + """ + text = Path(filepath).read_text() + info = {} + + m = re.search( + r"at xi,eta,gamma coordinates\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", + text, ) - parser.add_argument( - "--lowpass-period", - type=float, - default=None, - help="Override the lowpass filter period (in seconds). " - "Default uses the GF database anti-alias cutoff frequency.", + if m: + info["xi"] = float(m.group(1)) + info["eta"] = float(m.group(2)) + info["gamma"] = float(m.group(3)) + + m = re.search( + r"at \(x,y,z\)\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text ) - args = parser.parse_args() + if m: + info["x"] = float(m.group(1)) + info["y"] = float(m.group(2)) + info["z"] = float(m.group(3)) - gf_db = args.gf_db_path - fwd_dir = args.forward_output - station = args.station + for name in ["nu1", "nu2", "nu3"]: + m = re.search( + rf"{name}\s*=\s*([-\dE.+]+)\s+([-\dE.+]+)\s+([-\dE.+]+)", text + ) + if m: + info[name] = np.array( + [float(m.group(1)), float(m.group(2)), float(m.group(3))] + ) - if args.forward_solver_output: - solver_output = args.forward_solver_output - else: - solver_output = fwd_dir / "output_solver.txt" + m = re.search(r"latitude:\s*([-\dE.+]+)", text) + if m: + info["lat"] = float(m.group(1)) + m = re.search(r"longitude:\s*([-\dE.+]+)", text) + if m: + info["lon"] = float(m.group(1)) - if args.output: - output_plot = Path(args.output) - else: - output_plot = fwd_dir.parent / "gf_cross_validation.png" + m = re.search(r"East direction:\s*([-\dE.+]+)", text) + if m: + info["force_E"] = float(m.group(1)) + m = re.search(r"North direction:\s*([-\dE.+]+)", text) + if m: + info["force_N"] = float(m.group(1)) + m = re.search(r"Vertical direction:\s*([-\dE.+]+)", text) + if m: + info["force_Z"] = float(m.group(1)) - # --------------------------------------------------------------- - # 1. Parse forward simulation output - # --------------------------------------------------------------- - print("Parsing forward solver output...") - src = parse_solver_output(solver_output) - print(f" Source (x, y, z): ({src['x']:.8f}, {src['y']:.8f}, {src['z']:.8f})") - print(f" Source (xi, eta, gamma): ({src['xi']:.8f}, {src['eta']:.8f}, {src['gamma']:.8f})") - print(f" nu1 (N): {src['nu1']}") - print(f" nu2 (E): {src['nu2']}") - print(f" nu3 (Z): {src['nu3']}") - - # Determine which force component was used in forward simulation - force_dir = np.array([src.get("force_E", 0), src.get("force_N", 0), src.get("force_Z", 0)]) - print(f" Force direction (E, N, Z): {force_dir}") - - # Build the force direction in Cartesian from nu vectors - # force_cart = force_E * nu_E + force_N * nu_N + force_Z * nu_Z - # where nu_N = nu1, nu_E = nu2, nu_Z = nu3 + return info + + +def parse_cmtsolution(filepath): + """Parse CMTSOLUTION file for moment tensor components and source parameters. + + Returns dict with: Mrr, Mtt, Mpp, Mrt, Mrp, Mtp (dyne-cm), + half_duration, time_shift (seconds), latitude, longitude, depth (km). + """ + info = {} + with open(filepath) as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + if line.startswith("time shift:"): + info["time_shift"] = float(line.split(":")[1]) + elif line.startswith("half duration:"): + info["half_duration"] = float(line.split(":")[1]) + elif line.startswith("latitude:"): + info["latitude"] = float(line.split(":")[1]) + elif line.startswith("longitude:"): + info["longitude"] = float(line.split(":")[1]) + elif line.startswith("depth:"): + info["depth"] = float(line.split(":")[1]) + elif line.startswith("Mrr:"): + info["Mrr"] = float(line.split(":")[1]) + elif line.startswith("Mtt:"): + info["Mtt"] = float(line.split(":")[1]) + elif line.startswith("Mpp:"): + info["Mpp"] = float(line.split(":")[1]) + elif line.startswith("Mrt:"): + info["Mrt"] = float(line.split(":")[1]) + elif line.startswith("Mrp:"): + info["Mrp"] = float(line.split(":")[1]) + elif line.startswith("Mtp:"): + info["Mtp"] = float(line.split(":")[1]) + + return info + + +# --------------------------------------------------------------------------- +# Moment tensor rotation +# --------------------------------------------------------------------------- + +def rotate_mt_to_xyz(Mrr, Mtt, Mpp, Mrt, Mrp, Mtp, theta, phi): + """Rotate moment tensor from spherical (r, theta, phi) to Cartesian (x, y, z). + + Matches SPECFEM3D locate_sources.f90 lines 258-273 exactly. + """ + sint = sin(theta) + cost = cos(theta) + sinp = sin(phi) + cosp = cos(phi) + + Mxx = (sint*sint*cosp*cosp*Mrr + cost*cost*cosp*cosp*Mtt + sinp*sinp*Mpp + + 2.0*sint*cost*cosp*cosp*Mrt - 2.0*sint*sinp*cosp*Mrp + - 2.0*cost*sinp*cosp*Mtp) + + Myy = (sint*sint*sinp*sinp*Mrr + cost*cost*sinp*sinp*Mtt + cosp*cosp*Mpp + + 2.0*sint*cost*sinp*sinp*Mrt + 2.0*sint*sinp*cosp*Mrp + + 2.0*cost*sinp*cosp*Mtp) + + Mzz = cost*cost*Mrr + sint*sint*Mtt - 2.0*sint*cost*Mrt + + Mxy = (sint*sint*sinp*cosp*Mrr + cost*cost*sinp*cosp*Mtt - sinp*cosp*Mpp + + 2.0*sint*cost*sinp*cosp*Mrt + + sint*(cosp*cosp - sinp*sinp)*Mrp + + cost*(cosp*cosp - sinp*sinp)*Mtp) + + Mxz = (sint*cost*cosp*Mrr - sint*cost*cosp*Mtt + + (cost*cost - sint*sint)*cosp*Mrt + - cost*sinp*Mrp + sint*sinp*Mtp) + + Myz = (sint*cost*sinp*Mrr - sint*cost*sinp*Mtt + + (cost*cost - sint*sint)*sinp*Mrt + + cost*cosp*Mrp - sint*cosp*Mtp) + + return Mxx, Myy, Mzz, Mxy, Mxz, Myz + + +# --------------------------------------------------------------------------- +# Strain computation +# --------------------------------------------------------------------------- + +def compute_inverse_jacobian(xyz_elem, xi, eta, gamma): + """Compute the inverse Jacobian at a point within a spectral element.""" + Li = lagrange_basis(xi, GLL5) + Lj = lagrange_basis(eta, GLL5) + Lk = lagrange_basis(gamma, GLL5) + dLi = lagrange_deriv(xi, GLL5) + dLj = lagrange_deriv(eta, GLL5) + dLk = lagrange_deriv(gamma, GLL5) + + J = np.zeros((3, 3)) + for d in range(3): + J[d, 0] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], dLi, Lj, Lk) + J[d, 1] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, dLj, Lk) + J[d, 2] = np.einsum("kji,i,j,k->", xyz_elem[:, :, :, d], Li, Lj, dLk) + + return np.linalg.inv(J) + + +def compute_strain(displ, xi, eta, gamma, Jinv): + """Compute strain tensor from GF displacement at a point within an element. + + Parameters + ---------- + displ : ndarray, shape (nt, 5, 5, 5, 3, 3) + GF displacement on GLL nodes: (time, k, j, i, disp_comp, force_comp). + xi, eta, gamma : float + Reference coordinates of the target point. + Jinv : ndarray, shape (3, 3) + Inverse Jacobian: Jinv[ref_coord, phys_coord]. + + Returns + ------- + strain : ndarray, shape (nt, 6, 3) + Strain in Voigt notation [xx, yy, zz, xy, xz, yz] for each + force component (N, E, Z). + """ + Li = lagrange_basis(xi, GLL5) + Lj = lagrange_basis(eta, GLL5) + Lk = lagrange_basis(gamma, GLL5) + dLi = lagrange_deriv(xi, GLL5) + dLj = lagrange_deriv(eta, GLL5) + dLk = lagrange_deriv(gamma, GLL5) + + dG_dxi = np.einsum("tkjidb,i,j,k->tdb", displ, dLi, Lj, Lk) + dG_deta = np.einsum("tkjidb,i,j,k->tdb", displ, Li, dLj, Lk) + dG_dgamma = np.einsum("tkjidb,i,j,k->tdb", displ, Li, Lj, dLk) + + nt = displ.shape[0] + dGdx = np.zeros((nt, 3, 3, 3)) # (time, disp_comp, phys_deriv, force_comp) + for j in range(3): + dGdx[:, :, j, :] = (dG_dxi * Jinv[0, j] + + dG_deta * Jinv[1, j] + + dG_dgamma * Jinv[2, j]) + + # Symmetrize to get strain in Voigt notation: [xx, yy, zz, xy, xz, yz] + strain = np.zeros((nt, 6, 3)) + strain[:, 0, :] = dGdx[:, 0, 0, :] # eps_xx + strain[:, 1, :] = dGdx[:, 1, 1, :] # eps_yy + strain[:, 2, :] = dGdx[:, 2, 2, :] # eps_zz + strain[:, 3, :] = 0.5 * (dGdx[:, 0, 1, :] + dGdx[:, 1, 0, :]) # eps_xy + strain[:, 4, :] = 0.5 * (dGdx[:, 0, 2, :] + dGdx[:, 2, 0, :]) # eps_xz + strain[:, 5, :] = 0.5 * (dGdx[:, 1, 2, :] + dGdx[:, 2, 1, :]) # eps_yz + + return strain + + +# --------------------------------------------------------------------------- +# High-level reconstruction functions +# --------------------------------------------------------------------------- + +def read_mesh_info(gf_db): + """Read mesh timing parameters from the GF database. + + Returns dict with keys: dt, nstep, subsample_step, nt_sub. + """ + with h5py.File(gf_db / "mesh_info.h5", "r") as f: + info = { + "dt": float(np.asarray(f.attrs["dt"]).flat[0]), + "nstep": int(np.asarray(f.attrs["nstep"]).flat[0]), + "subsample_step": int(np.asarray(f.attrs["subsample_step"]).flat[0]), + "nt_sub": int(np.asarray(f.attrs["nt_subsampled"]).flat[0]), + } + return info + + +def read_station_meta(gf_db, station): + """Read station metadata from the GF database. + + Returns dict with keys: f_cutoff, hdur, factor_force_nondim. + """ + with h5py.File(gf_db / "stations" / f"{station}.h5", "r") as f: + meta = { + "f_cutoff": float(np.asarray(f.attrs["f_cutoff"]).flat[0]), + "hdur": float(np.asarray(f.attrs["hdur"]).flat[0]), + "factor_force_nondim": float( + np.asarray(f.attrs["factor_force_source"]).flat[0] + ), + } + return meta + + +def reconstruct_force(displ, xi, eta, gamma, src): + """Reconstruct seismograms from GF database using force source reciprocity. + + Returns ndarray of shape (3, nt) with components [N, E, Z]. + """ nu_force = ( src.get("force_N", 0) * src["nu1"] + src.get("force_E", 0) * src["nu2"] + src.get("force_Z", 0) * src["nu3"] ) - print(f" Force direction (Cartesian): {nu_force}") - # --------------------------------------------------------------- - # 2. Read mesh info - # --------------------------------------------------------------- - print("\nReading mesh info...") - with h5py.File(gf_db / "mesh_info.h5", "r") as f: - dt = float(np.asarray(f.attrs["dt"]).flat[0]) - nstep = int(np.asarray(f.attrs["nstep"]).flat[0]) - subsample_step = int(np.asarray(f.attrs["subsample_step"]).flat[0]) - nt_sub = int(np.asarray(f.attrs["nt_subsampled"]).flat[0]) - print(f" dt={dt}, nstep={nstep}, subsample_step={subsample_step}, nt_sub={nt_sub}") - - # Read station metadata for STF filter params - sta_file = gf_db / "stations" / f"{station}.h5" - with h5py.File(sta_file, "r") as f: - f_cutoff = float(np.asarray(f.attrs["f_cutoff"]).flat[0]) - hdur = float(np.asarray(f.attrs["hdur"]).flat[0]) - print(f" f_cutoff={f_cutoff:.4f} Hz, hdur={hdur:.6f} s") - - # --------------------------------------------------------------- - # 3. Find containing element - # --------------------------------------------------------------- - print("\nFinding containing element...") - source_xyz = np.array([src["x"], src["y"], src["z"]]) - morton_hex, xi, eta, gamma, dist = find_containing_element(gf_db, source_xyz) - print(f" Containing element: {morton_hex} (centroid dist: {dist:.6e})") - print(f" Located at (xi={xi:.6f}, eta={eta:.6f}, gamma={gamma:.6f})") + G = interpolate_gll(displ, xi, eta, gamma) # (nt, 3_disp, 3_force) - # --------------------------------------------------------------- - # 4. Read GF displacement and interpolate - # --------------------------------------------------------------- - print("\nReading GF displacement...") - gf_file = gf_db / "elements" / morton_hex / f"{station}.h5" - with h5py.File(gf_file, "r") as f: - # Python shape: (nt_sub, NGLLZ, NGLLY, NGLLX, 3_disp, 3_force) - displ = f["displacement"][:] - print(f" Displacement shape: {displ.shape}, dtype: {displ.dtype}") - - # Interpolate to the exact source point using reference coordinates - print(f" Interpolating at (xi={xi:.6f}, eta={eta:.6f}, gamma={gamma:.6f})...") - - # displ shape: (nt, k, j, i, disp_comp, force_comp) - # We want to contract over k, j, i dimensions (indices 1, 2, 3) - # Result: G[nt, disp_comp, force_comp] - G = interpolate_gll(displ, xi, eta, gamma) - print(f" Green tensor shape: {G.shape}") # should be (nt_sub, 3, 3) - - # --------------------------------------------------------------- - # 5. Apply reciprocity - # --------------------------------------------------------------- - # G[t, disp_xyz, force_NEZ] is the Cartesian displacement at the source - # from a geographic force at the station. - # - # By reciprocity, the geographic displacement at the station from a - # Cartesian force F_cart at the source is: - # u_geo_b(station) = sum_j F_cart_j * G[t, j, b] - # - # Our forward force is in geographic direction -> Cartesian via nu: - # F_cart = force_E * nu_E + force_N * nu_N + force_Z * nu_Z = nu_force - # - # So: u_b(station) = nu_force . G[:, :, b] - - print("\nApplying reciprocity...") - gf_N = G[:, :, 0] @ nu_force # geographic N at station - gf_E = G[:, :, 1] @ nu_force # geographic E at station - gf_Z = G[:, :, 2] @ nu_force # geographic Z at station - print(f" GF trace ranges: N=[{gf_N.min():.6e}, {gf_N.max():.6e}], " - f"E=[{gf_E.min():.6e}, {gf_E.max():.6e}], " - f"Z=[{gf_Z.min():.6e}, {gf_Z.max():.6e}]") - - # --------------------------------------------------------------- - # 6. Convolve GF reconstruction with corrected STF - # --------------------------------------------------------------- - # The GF database has hdur_gf baked into the stored displacement. - # The forward simulation uses hdur_fwd (from f0 / SOURCE_DECAY_MIMIC_TRIANGLE). - # By Tarantola (2005, eq. 6.21), the convolution of two Gaussians gives a - # Gaussian with sigma_total = sqrt(sigma_1^2 + sigma_2^2). - # So we convolve the GF reconstruction with a corrected Gaussian: - # hdur_corrected = sqrt(hdur_fwd^2 - hdur_gf^2) - # to produce the equivalent of the forward trace. - - SOURCE_DECAY_MIMIC_TRIANGLE = 1.628 - - # Parse forward hdur from solver output - fwd_text = Path(solver_output).read_text() - m = re.search(r"Gaussian half duration:\s*([\d.Ee+-]+)", fwd_text) - if m: - hdur_fwd = float(m.group(1)) - else: - # Fallback: read f0 from FORCESOLUTION - m2 = re.search(r"half duration:\s*([\d.Ee+-]+)\s*seconds", fwd_text) - hdur_fwd = float(m2.group(1)) / SOURCE_DECAY_MIMIC_TRIANGLE if m2 else hdur + gf = np.zeros((3, G.shape[0])) + for i in range(3): + gf[i] = G[:, :, i] @ nu_force + return gf - hdur_gf = hdur # from station metadata - print(f"\nSTF correction (Tarantola 2005, eq. 6.21):") - print(f" hdur_fwd (forward Gaussian): {hdur_fwd:.6f} s") - print(f" hdur_gf (GF database): {hdur_gf:.6f} s") +def reconstruct_cmt(displ, xi, eta, gamma, gf_db, morton_hex, + src, cmt_path, factor_force_nondim, dt_sub): + """Reconstruct seismograms from GF database using moment tensor source. - if hdur_fwd > hdur_gf: - hdur_corrected = np.sqrt(hdur_fwd**2 - hdur_gf**2) - else: - print(" WARNING: hdur_fwd <= hdur_gf, no correction needed") - hdur_corrected = 0.0 + Computes strain, rotates the MT, contracts, and time-integrates + to convert from Gaussian to Heaviside STF. + + Returns ndarray of shape (3, nt) with components [N, E, Z]. + """ + # Inverse Jacobian + coord_file = gf_db / "elements" / morton_hex / "coordinates.h5" + with h5py.File(coord_file, "r") as f: + xyz_elem = f["xyz"][:] + Jinv = compute_inverse_jacobian(xyz_elem, xi, eta, gamma) + + # Strain from displacement gradients + strain = compute_strain(displ, xi, eta, gamma, Jinv) # (nt, 6, 3) + + # Parse CMTSOLUTION + cmt = parse_cmtsolution(cmt_path) + + # Rotate moment tensor from spherical to Cartesian + r = sqrt(src["x"]**2 + src["y"]**2 + src["z"]**2) + theta = np.arccos(src["z"] / r) + phi = np.arctan2(src["y"], src["x"]) + + Mxx, Myy, Mzz, Mxy, Mxz, Myz = rotate_mt_to_xyz( + cmt["Mrr"], cmt["Mtt"], cmt["Mpp"], + cmt["Mrt"], cmt["Mrp"], cmt["Mtp"], + theta, phi, + ) - print(f" hdur_corrected: {hdur_corrected:.6f} s") + # Contract strain with moment tensor (Voigt notation) + voigt_factor = np.array([1.0, 1.0, 1.0, 2.0, 2.0, 2.0]) + Mx = np.array([Mxx, Myy, Mzz, Mxy, Mxz, Myz]) + mt_scale = 1.0 / (SCALE_M * factor_force_nondim) + weights = voigt_factor * Mx * mt_scale # shape (6,) - # Build corrected Gaussian STF and convolve with GF traces - # Time axis for GF (subsampled dt) - dt_sub = dt * subsample_step - # STF kernel: symmetric Gaussian centered at t=0 - # Extent: +-4*hdur_corrected should be sufficient - if hdur_corrected > 0: - stf_half_len = int(4.0 * hdur_corrected / dt_sub) - stf_t = np.arange(-stf_half_len, stf_half_len + 1) * dt_sub - stf = np.exp(-(stf_t / hdur_corrected)**2) / (np.sqrt(pi) * hdur_corrected) - stf *= dt_sub # normalize for discrete convolution - - print(f" STF kernel: {len(stf)} samples, peak={stf.max():.6e}") - - # Convolve GF traces with corrected STF using FFT for proper handling. - # The STF is causal-shifted: we want the peak of the Gaussian at t=0 - # to correspond to no time shift in the output. - from scipy.signal import fftconvolve - gf_N = fftconvolve(gf_N, stf, mode="same") - gf_E = fftconvolve(gf_E, stf, mode="same") - gf_Z = fftconvolve(gf_Z, stf, mode="same") - print(f" After convolution - GF trace ranges:") - print(f" N=[{gf_N.min():.6e}, {gf_N.max():.6e}]") - print(f" E=[{gf_E.min():.6e}, {gf_E.max():.6e}]") - print(f" Z=[{gf_Z.min():.6e}, {gf_Z.max():.6e}]") - - # --------------------------------------------------------------- - # 7. Read and filter forward seismograms - # --------------------------------------------------------------- - print("\nReading forward seismograms...") - fwd_traces = {} - for comp in ["BXN", "BXE", "BXZ"]: + gf = np.zeros((3, strain.shape[0])) + for i in range(3): + gf[i] = np.sum(weights[:, None] * strain[:, :, i].T, axis=0) + + # Time-integrate: Gaussian -> Heaviside STF conversion + for i in range(3): + gf[i] = np.cumsum(gf[i]) * dt_sub + + return gf, cmt + + +def apply_stf_correction(gf, dt_sub, hdur_fwd, hdur_gf): + """Convolve GF reconstruction with corrected source time function. + + Applies Tarantola (2005) eq. 6.21 STF correction in-place. + + Parameters + ---------- + gf : ndarray, shape (3, nt) + GF traces to correct. + dt_sub : float + Subsampled time step. + hdur_fwd : float + Forward simulation Gaussian half-duration. + hdur_gf : float + GF database half-duration. + """ + if hdur_fwd <= hdur_gf: + print(" WARNING: hdur_fwd <= hdur_gf, no STF correction applied") + return + + hdur_corrected = np.sqrt(hdur_fwd**2 - hdur_gf**2) + stf_half_len = int(4.0 * hdur_corrected / dt_sub) + stf_t = np.arange(-stf_half_len, stf_half_len + 1) * dt_sub + stf = np.exp(-(stf_t / hdur_corrected)**2) / (np.sqrt(pi) * hdur_corrected) + stf *= dt_sub + + print(f" STF correction: hdur_corrected={hdur_corrected:.4f} s, " + f"kernel={len(stf)} samples") + + for i in range(3): + gf[i] = fftconvolve(gf[i], stf, mode="same") + + +def read_forward_seismograms(fwd_dir, station): + """Read forward simulation seismograms (auto-detect channel codes). + + Returns + ------- + traces : dict + Keys are channel codes (e.g. 'BXN'), values are dicts with + 'time' (ndarray) and 'amp' (ndarray, float64). + channel_prefix : str + Detected prefix (e.g. 'BX', 'BH', 'MX'). + """ + for prefix in ["BX", "BH", "MX"]: + test_file = fwd_dir / f"{station}.{prefix}N.sem.sac" + if test_file.exists(): + channel_prefix = prefix + break + else: + print("ERROR: Could not find forward seismogram files.") + sys.exit(1) + + traces = {} + for suffix in ["N", "E", "Z"]: + comp = f"{channel_prefix}{suffix}" sac_file = fwd_dir / f"{station}.{comp}.sem.sac" tr = obspy_read(str(sac_file))[0] - time = tr.times() + tr.stats.sac.b # seconds from origin time - fwd_traces[comp] = {"time": time, "amp": tr.data.astype(np.float64)} - print(f" {comp}: {len(tr.data)} samples, " - f"amp range [{tr.data.min():.6e}, {tr.data.max():.6e}]") - - # Apply Butterworth filter to match GF anti-alias filtering, then subsample - lp_fc = 1.0 / args.lowpass_period if args.lowpass_period else f_cutoff - lp_filter = args.lowpass_period is not None - print(f"\nApplying Butterworth lowpass filter (order=4, fc={lp_fc:.4f} Hz" - f" [T={1/lp_fc:.1f}s], fs={1/dt:.4f} Hz)...") + time = tr.times() + tr.stats.sac.b + traces[comp] = {"time": time, "amp": tr.data.astype(np.float64)} + + return traces, channel_prefix + + +def align_and_filter(gf, time_gf, fwd_traces, dt, subsample_step, dt_sub, + f_cutoff, lowpass_period, highpass_period): + """Lowpass-filter forward traces, subsample, align with GF, and apply optional filters. + + Parameters + ---------- + gf : ndarray, shape (3, nt_gf) + GF reconstruction [N, E, Z]. + time_gf : ndarray + GF time axis. + fwd_traces : dict + Forward seismogram traces from read_forward_seismograms(). + dt, subsample_step, dt_sub : float + Timing parameters. + f_cutoff : float + GF database anti-alias cutoff frequency (Hz). + lowpass_period, highpass_period : float or None + Optional filter periods. + + Returns + ------- + time_common : ndarray + Common time axis after alignment. + gf_aligned : dict + GF traces keyed by channel code. + fwd_aligned : dict + Forward traces keyed by channel code. + comps : list + Channel code list ['BXN', 'BXE', 'BXZ']. + comp_labels : dict + Mapping from channel code to label name. + """ + comps = list(fwd_traces.keys()) + comp_labels = {comps[0]: "North", comps[1]: "East", comps[2]: "Vertical"} + + # Lowpass filter forward traces to match GF anti-alias + lp_fc = 1.0 / lowpass_period if lowpass_period else f_cutoff + lp_filter = lowpass_period is not None sos = butterworth_sos(4, lp_fc, 1.0 / dt) fwd_filtered = {} - for comp in ["BXN", "BXE", "BXZ"]: - amp = fwd_traces[comp]["amp"].astype(np.float64) - filtered = filter_with_taper(amp, sos) - fwd_filtered[comp] = filtered + for comp in comps: + fwd_filtered[comp] = filter_with_taper( + fwd_traces[comp]["amp"].astype(np.float64), sos + ) # Subsample forward traces - fwd_sub = {} - for comp in ["BXN", "BXE", "BXZ"]: - fwd_sub[comp] = fwd_filtered[comp][::subsample_step] - time_fwd = fwd_traces["BXN"]["time"][::subsample_step] - - # Construct GF time axis - # GF stores at it = subsample_step, 2*subsample_step, ... - # Time for snapshot isnap (1-based): t = (isnap * subsample_step - 1) * dt - t0_gf - # Parse t0_gf from GF solver output - t0_gf = None - gf_solver_candidates = [ - args.gf_solver_output, - fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver.txt", - fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver_N.txt", - ] - for gf_solver_path in gf_solver_candidates: - if gf_solver_path is not None and gf_solver_path.exists(): - gf_solver_text = gf_solver_path.read_text() - m = re.search(r"start time\s*:\s*([-\dE.+]+)", gf_solver_text) - if m: - t0_gf = -float(m.group(1)) # start time is -t0 - print(f" Found GF t0 from: {gf_solver_path}") - break - if t0_gf is None: - # Fallback: t0_gf = T_min_period / 2 - t0_gf = hdur_gf * 5.0 # approximate - print(f" WARNING: could not find GF t0, estimating t0_gf={t0_gf:.4f}") - - dt_sub = dt * subsample_step - time_gf = np.array([(isnap * subsample_step - 1) * dt - t0_gf - for isnap in range(1, nt_sub + 1)]) - - print(f" Forward subsampled: {len(time_fwd)} samples, t=[{time_fwd[0]:.2f}, {time_fwd[-1]:.2f}]") - print(f" GF: {nt_sub} samples, t=[{time_gf[0]:.2f}, {time_gf[-1]:.2f}]") - print(f" t0_gf={t0_gf:.4f}, t0_fwd={-fwd_traces['BXN']['time'][0]:.4f}") + fwd_sub = {comp: fwd_filtered[comp][::subsample_step] for comp in comps} + time_fwd = fwd_traces[comps[0]]["time"][::subsample_step] # Align traces using overlapping time window - # Find common time range t_start = max(time_fwd[0], time_gf[0]) t_end = min(time_fwd[-1], time_gf[-1]) - print(f" Common time window: [{t_start:.2f}, {t_end:.2f}] s") - # Use GF time axis as reference (it has fewer samples) - # Interpolate forward trace onto GF time points within the common window mask_gf = (time_gf >= t_start) & (time_gf <= t_end) time_common = time_gf[mask_gf] - gf_traces = { - "BXN": gf_N[mask_gf], - "BXE": gf_E[mask_gf], - "BXZ": gf_Z[mask_gf], + gf_aligned = { + comps[0]: gf[0][mask_gf], + comps[1]: gf[1][mask_gf], + comps[2]: gf[2][mask_gf], } - fwd_interp = {} - for comp in ["BXN", "BXE", "BXZ"]: - fwd_interp[comp] = np.interp(time_common, time_fwd, fwd_sub[comp]) + fwd_aligned = {} + for comp in comps: + fwd_aligned[comp] = np.interp(time_common, time_fwd, fwd_sub[comp]) + # Taper both signals to suppress edge artifacts n_compare = len(time_common) - print(f" Comparison samples: {n_compare}") - - # --------------------------------------------------------------- - # 7b. Optional lowpass filter on GF traces - # --------------------------------------------------------------- - # The forward traces are always lowpass-filtered before subsampling (section 7). - # The GF traces already have the database anti-alias filter baked in. - # When --lowpass-period is explicitly set, apply the same lowpass to GF traces - # so both are filtered identically. + n_taper = max(1, int(0.15 * n_compare)) + taper = np.ones(n_compare) + taper[:n_taper] = 0.5 * (1.0 - np.cos(np.linspace(0, pi, n_taper))) + taper[-n_taper:] = 0.5 * (1.0 - np.cos(np.linspace(pi, 0, n_taper))) + for comp in comps: + gf_aligned[comp] = gf_aligned[comp] * taper + fwd_aligned[comp] = fwd_aligned[comp] * taper + + # Optional lowpass filter on GF traces (when overriding database cutoff) if lp_filter: - lp_fc_sub = 1.0 / args.lowpass_period - lp_fs_sub = 1.0 / dt_sub - lp_order = 4 - print(f"\nApplying Butterworth lowpass filter to GF traces (order={lp_order}, " - f"period={args.lowpass_period:.1f} s, fc={lp_fc_sub:.6f} Hz, fs={lp_fs_sub:.4f} Hz)...") - lp_sos_sub = butterworth_sos(lp_order, lp_fc_sub, lp_fs_sub) - for comp in ["BXN", "BXE", "BXZ"]: - gf_traces[comp] = filter_with_taper(gf_traces[comp], lp_sos_sub) - - # --------------------------------------------------------------- - # 7c. Optional highpass filter - # --------------------------------------------------------------- - if args.highpass_period is not None: - hp_fc = 1.0 / args.highpass_period # convert period to frequency - hp_fs = 1.0 / dt_sub # sampling rate of subsampled traces - hp_order = 4 - print(f"\nApplying Butterworth highpass filter (order={hp_order}, " - f"period={args.highpass_period:.1f} s, fc={hp_fc:.6f} Hz, fs={hp_fs:.4f} Hz)...") - hp_sos = butterworth_highpass_sos(hp_order, hp_fc, hp_fs) - for comp in ["BXN", "BXE", "BXZ"]: - gf_traces[comp] = filter_with_taper(gf_traces[comp], hp_sos) - fwd_interp[comp] = filter_with_taper(fwd_interp[comp], hp_sos) - - # --------------------------------------------------------------- - # 8. Compute residuals - # --------------------------------------------------------------- - print("\nResiduals:") - for comp in ["BXN", "BXE", "BXZ"]: - fwd = fwd_interp[comp] - gf = gf_traces[comp] - residual = np.sqrt(np.sum((fwd - gf) ** 2)) - ref = np.sqrt(np.sum(fwd ** 2)) - rel_err = residual / ref if ref > 0 else float("inf") - print(f" {comp}: ||fwd - gf|| / ||fwd|| = {rel_err:.6e} " - f"(abs: {residual:.6e}, ref: {ref:.6e})") - - # --------------------------------------------------------------- - # 9. Plot comparison - # --------------------------------------------------------------- - print(f"\nPlotting to {output_plot}...") - fig, axes = plt.subplots(3, 1, figsize=(14, 10), sharex=True) - comp_labels = {"BXN": "North", "BXE": "East", "BXZ": "Vertical"} + lp_sos_sub = butterworth_sos(4, 1.0 / lowpass_period, 1.0 / dt_sub) + for comp in comps: + gf_aligned[comp] = filter_with_taper(gf_aligned[comp], lp_sos_sub) + + # Optional highpass filter on both + if highpass_period is not None: + hp_sos = butterworth_highpass_sos(4, 1.0 / highpass_period, 1.0 / dt_sub) + for comp in comps: + gf_aligned[comp] = filter_with_taper(gf_aligned[comp], hp_sos) + fwd_aligned[comp] = filter_with_taper(fwd_aligned[comp], hp_sos) + + return time_common, gf_aligned, fwd_aligned, comps, comp_labels - for ax, comp in zip(axes, ["BXN", "BXE", "BXZ"]): - fwd = fwd_interp[comp] + +def compute_residuals(gf_traces, fwd_traces, comps): + """Compute relative RMS error per component. + + Returns dict of {comp: rel_err}. + """ + residuals = {} + for comp in comps: + fwd = fwd_traces[comp] gf = gf_traces[comp] + rms_err = np.sqrt(np.sum((fwd - gf) ** 2)) + rms_ref = np.sqrt(np.sum(fwd ** 2)) + residuals[comp] = rms_err / rms_ref if rms_ref > 0 else float("inf") + return residuals - residual = np.sqrt(np.sum((fwd - gf) ** 2)) - ref = np.sqrt(np.sum(fwd ** 2)) - rel_err = residual / ref if ref > 0 else float("inf") - ax.plot(time_common, fwd, "b-", linewidth=0.8, label="Forward (filtered)") - ax.plot(time_common, gf, "r--", linewidth=0.8, label="GF reconstructed") +def plot_comparison(time_common, gf_traces, fwd_traces, comps, comp_labels, + residuals, station, source_type, morton_hex, xi, eta, gamma, + highpass_period, lowpass_period, output_plot): + """Plot GF vs forward comparison and save.""" + fig, axes = plt.subplots(3, 1, figsize=(14, 10), sharex=True) + + for ax, comp in zip(axes, comps): + ax.plot(time_common, fwd_traces[comp], "b-", linewidth=0.8, + label="Forward (filtered)") + ax.plot(time_common, gf_traces[comp], "r--", linewidth=0.8, + label="GF reconstructed") ax.set_ylabel(f"{comp} ({comp_labels[comp]})") - ax.set_title(f"{comp} — relative error: {rel_err:.4e}") + ax.set_title(f"{comp} — relative error: {residuals[comp]:.4e}") ax.legend(loc="upper right", fontsize=8) ax.grid(True, alpha=0.3) axes[-1].set_xlabel("Time (s)") + filter_info = "" - if args.lowpass_period: - filter_info += f", lowpass T<{args.lowpass_period:.0f}s" - if args.highpass_period: - filter_info += f", highpass T>{args.highpass_period:.0f}s" + if lowpass_period: + filter_info += f", lowpass T<{lowpass_period:.0f}s" + if highpass_period: + filter_info += f", highpass T>{highpass_period:.0f}s" + fig.suptitle( - f"GF Cross-Validation: {station}{filter_info}\n" + f"GF Cross-Validation ({source_type}): {station}{filter_info}\n" f"Source at element {morton_hex}\n" f"(xi={xi:.4f}, eta={eta:.4f}, gamma={gamma:.4f})", fontsize=11, ) plt.tight_layout() plt.savefig(output_plot, dpi=150) - print(f"Saved {output_plot}") plt.close() +# --------------------------------------------------------------------------- +# Main cross-validation logic +# --------------------------------------------------------------------------- + +def cross_validate( + gf_db_path, + forward_output, + station, + force=False, + cmtsolution=None, + highpass_period=None, + lowpass_period=None, + output=None, + forward_solver_output=None, + gf_solver_output=None, +): + """Cross-validate a GF database against a forward simulation. + + Parameters + ---------- + gf_db_path : Path + Path to the GF database directory. + forward_output : Path + Path to the forward simulation OUTPUT_FILES directory. + station : str + Station name as NET.STA. + force : bool + If True, use force source workflow; otherwise use CMT. + cmtsolution : Path or None + Path to CMTSOLUTION file. Auto-detected if None and force=False. + highpass_period : float or None + Highpass filter period in seconds. + lowpass_period : float or None + Override lowpass filter period in seconds. + output : Path or None + Output plot path. Auto-generated if None. + forward_solver_output : Path or None + Path to forward output_solver.txt. Auto-detected if None. + gf_solver_output : Path or None + Path to GF output_solver.txt for t0_gf. Auto-detected if None. + + Returns + ------- + output_plot : Path + Path to the saved plot. + """ + gf_db = Path(gf_db_path) + fwd_dir = Path(forward_output) + + solver_output = forward_solver_output or fwd_dir / "output_solver.txt" + output_plot = Path(output) if output else fwd_dir.parent / "gf_cross_validation.png" + + source_type = "Force" if force else "CMT" + print(f"\nSource type: {source_type}") + + # 1. Parse forward simulation output + print("Parsing forward solver output...") + src = parse_solver_output(solver_output) + print(f" Source (x,y,z): ({src['x']:.6f}, {src['y']:.6f}, {src['z']:.6f})") + + # 2. Read mesh info and station metadata + mesh = read_mesh_info(gf_db) + dt = mesh["dt"] + subsample_step = mesh["subsample_step"] + nt_sub = mesh["nt_sub"] + dt_sub = dt * subsample_step + + sta_meta = read_station_meta(gf_db, station) + print(f" f_cutoff={sta_meta['f_cutoff']:.4f} Hz, hdur={sta_meta['hdur']:.6f} s") + + # 3. Find containing element + print("Finding containing element...") + source_xyz = np.array([src["x"], src["y"], src["z"]]) + morton_hex, xi, eta, gamma, dist = find_containing_element(gf_db, source_xyz) + print(f" Element: {morton_hex} (xi={xi:.4f}, eta={eta:.4f}, gamma={gamma:.4f})") + + # 4. Read GF displacement + gf_file = gf_db / "elements" / morton_hex / f"{station}.h5" + with h5py.File(gf_file, "r") as f: + displ = f["displacement"][:] + + # 5. Reconstruct seismograms + if force: + gf = reconstruct_force(displ, xi, eta, gamma, src) + else: + # Auto-detect CMTSOLUTION if not provided + if cmtsolution is None: + candidates = [ + fwd_dir.parent / "DATA" / "CMTSOLUTION", + fwd_dir / ".." / "DATA" / "CMTSOLUTION", + ] + for c in candidates: + if c.resolve().exists(): + cmtsolution = c.resolve() + break + if cmtsolution is None: + print("ERROR: Could not find CMTSOLUTION file. " + "Use --cmtsolution to specify.") + sys.exit(1) + + gf, cmt = reconstruct_cmt( + displ, xi, eta, gamma, gf_db, morton_hex, + src, cmtsolution, sta_meta["factor_force_nondim"], dt_sub, + ) + + # 6. STF correction + if force: + fwd_text = Path(solver_output).read_text() + m = re.search(r"Gaussian half duration:\s*([\d.Ee+-]+)", fwd_text) + if m: + hdur_fwd = float(m.group(1)) + else: + m2 = re.search(r"half duration:\s*([\d.Ee+-]+)\s*seconds", fwd_text) + hdur_fwd = (float(m2.group(1)) / SOURCE_DECAY_MIMIC_TRIANGLE + if m2 else sta_meta["hdur"]) + else: + hdur_fwd = cmt["half_duration"] / SOURCE_DECAY_MIMIC_TRIANGLE + + apply_stf_correction(gf, dt_sub, hdur_fwd, sta_meta["hdur"]) + + # 7. Read forward seismograms + fwd_traces, channel_prefix = read_forward_seismograms(fwd_dir, station) + + # 8. Construct GF time axis + t0_gf = None + gf_solver_candidates = [ + gf_solver_output, + fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver.txt", + fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver_N.txt", + ] + for gf_solver_path in gf_solver_candidates: + if gf_solver_path is not None and Path(gf_solver_path).exists(): + gf_solver_text = Path(gf_solver_path).read_text() + m = re.search(r"start time\s*:\s*([-\dE.+]+)", gf_solver_text) + if m: + t0_gf = -float(m.group(1)) + break + if t0_gf is None: + t0_gf = sta_meta["hdur"] * 5.0 + print(f" WARNING: could not find GF t0, estimating t0_gf={t0_gf:.4f}") + + time_gf = np.array([(isnap * subsample_step - 1) * dt - t0_gf + for isnap in range(1, nt_sub + 1)]) + + # 9. Align, filter, compare + time_common, gf_aligned, fwd_aligned, comps, comp_labels = align_and_filter( + gf, time_gf, fwd_traces, dt, subsample_step, dt_sub, + sta_meta["f_cutoff"], lowpass_period, highpass_period, + ) + + # 10. Compute and print residuals + residuals = compute_residuals(gf_aligned, fwd_aligned, comps) + print("Residuals:") + for comp in comps: + print(f" {comp}: {residuals[comp]:.6e}") + + # 11. Plot + output_plot.parent.mkdir(parents=True, exist_ok=True) + plot_comparison( + time_common, gf_aligned, fwd_aligned, comps, comp_labels, + residuals, station, source_type, morton_hex, xi, eta, gamma, + highpass_period, lowpass_period, output_plot, + ) + print(f"Saved {output_plot}") + + return output_plot + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="Cross-validate GF database against forward simulation" + ) + parser.add_argument( + "gf_db_path", type=Path, help="Path to GF database directory" + ) + parser.add_argument( + "forward_output", type=Path, help="Path to forward OUTPUT_FILES directory" + ) + parser.add_argument( + "--forward-solver-output", type=Path, default=None, + help="Path to forward output_solver.txt " + "(default: /output_solver.txt)", + ) + parser.add_argument( + "--station", type=str, default="IU.SJG", + help="Station name as NET.STA (default: IU.SJG)", + ) + parser.add_argument( + "--gf-solver-output", type=Path, default=None, + help="Path to GF output_solver.txt (to get t0_gf; default: auto-detect)", + ) + parser.add_argument( + "--output", type=str, default=None, + help="Output plot path (default: gf_cross_validation.png next to forward output)", + ) + parser.add_argument( + "--force", action="store_true", default=False, + help="Use force source workflow (default: moment tensor / CMT workflow).", + ) + parser.add_argument( + "--cmtsolution", type=Path, default=None, + help="Path to CMTSOLUTION file " + "(default: auto-detect from forward output directory).", + ) + parser.add_argument( + "--highpass-period", type=float, default=None, + help="Apply zero-phase Butterworth highpass filter at this period (seconds).", + ) + parser.add_argument( + "--lowpass-period", type=float, default=None, + help="Override the lowpass filter period (seconds). " + "Default uses the GF database anti-alias cutoff frequency.", + ) + args = parser.parse_args() + + cross_validate( + gf_db_path=args.gf_db_path, + forward_output=args.forward_output, + station=args.station, + force=args.force, + cmtsolution=args.cmtsolution, + highpass_period=args.highpass_period, + lowpass_period=args.lowpass_period, + output=Path(args.output) if args.output else None, + forward_solver_output=args.forward_solver_output, + gf_solver_output=args.gf_solver_output, + ) + + if __name__ == "__main__": main() diff --git a/utils/green_function/xvalidate.py b/utils/green_function/xvalidate.py new file mode 100644 index 000000000..cb6ee0fab --- /dev/null +++ b/utils/green_function/xvalidate.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Cross-validate a Green function database against forward simulations. + +Runs both force and CMT validation from a single base directory, +auto-deriving all parameters from the standard directory layout. + +Expected directory structure: + basedir/ + ├── db_base/DATA/STATIONS (station list for GF database) + ├── db_base/OUTPUT_FILES/values_from_mesher.h (min resolved period) + ├── GFDB/ (GF database) + ├── forward/OUTPUT_FILES/ (force validation forward sim) + ├── forward_cmt/OUTPUT_FILES/ (CMT validation forward sim) + └── validation_data/CMTSOLUTION (CMT source for validation) + +Usage: + python xvalidate.py EXAMPLES/green_function_database/regional +""" + +import argparse +import re +import sys +from pathlib import Path + +# Import cross_validate from sibling module +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from gf_cross_validate import cross_validate + + +def parse_station_from_file(stations_path): + """Read the first station from a SPECFEM STATIONS file. + + Returns NET.STA string (e.g. 'IU.SJG'). + """ + with open(stations_path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) >= 2: + return f"{parts[1]}.{parts[0]}" + raise ValueError(f"No station found in {stations_path}") + + +def parse_min_period(values_from_mesher_path): + """Extract minimum resolved period from values_from_mesher.h. + + Parses the line: + ! the (approximate) minimum period resolved will be = 69.4968262 (s) + """ + text = values_from_mesher_path.read_text() + m = re.search(r"minimum period resolved will be\s*=\s*([\d.Ee+-]+)", text) + if m: + return float(m.group(1)) + raise ValueError( + f"Could not parse minimum period from {values_from_mesher_path}" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Cross-validate GF database (force + CMT) from base directory" + ) + parser.add_argument( + "basedir", type=Path, + help="Base directory (e.g., EXAMPLES/green_function_database/regional)", + ) + parser.add_argument( + "--force-only", action="store_true", + help="Run only the force source validation", + ) + parser.add_argument( + "--cmt-only", action="store_true", + help="Run only the CMT validation", + ) + args = parser.parse_args() + + basedir = args.basedir.resolve() + + # Derive all paths from directory structure + gf_db_path = basedir / "GFDB" + forward_force_output = basedir / "forward" / "OUTPUT_FILES" + forward_cmt_output = basedir / "forward_cmt" / "OUTPUT_FILES" + cmtsolution = basedir / "validation_data" / "CMTSOLUTION" + stations_file = basedir / "db_base" / "DATA" / "STATIONS" + values_from_mesher = basedir / "db_base" / "OUTPUT_FILES" / "values_from_mesher.h" + output_dir = basedir / "validation_output" + + # Validate required paths + missing = [] + for label, path in [ + ("GF database", gf_db_path), + ("STATIONS file", stations_file), + ("values_from_mesher.h", values_from_mesher), + ]: + if not path.exists(): + missing.append(f" {label}: {path}") + + if not args.force_only: + if not forward_cmt_output.exists(): + missing.append(f" CMT forward output: {forward_cmt_output}") + if not cmtsolution.exists(): + missing.append(f" CMTSOLUTION: {cmtsolution}") + + if not args.cmt_only: + if not forward_force_output.exists(): + missing.append(f" Force forward output: {forward_force_output}") + + if missing: + print("ERROR: Missing required paths:") + print("\n".join(missing)) + sys.exit(1) + + # Parse derived parameters + station = parse_station_from_file(stations_file) + highpass_period = parse_min_period(values_from_mesher) + + print(f"Base directory: {basedir}") + print(f"Station: {station}") + print(f"Highpass period: {highpass_period:.1f} s (from mesh)") + + output_dir.mkdir(exist_ok=True) + + # Run force validation + if not args.cmt_only: + print("\n" + "=" * 60) + print("FORCE SOURCE VALIDATION") + print("=" * 60) + cross_validate( + gf_db_path=gf_db_path, + forward_output=forward_force_output, + station=station, + force=True, + highpass_period=highpass_period, + output=output_dir / "xvalidate_force.svg", + ) + + # Run CMT validation + if not args.force_only: + print("\n" + "=" * 60) + print("CMT SOURCE VALIDATION") + print("=" * 60) + cross_validate( + gf_db_path=gf_db_path, + forward_output=forward_cmt_output, + station=station, + force=False, + cmtsolution=cmtsolution, + highpass_period=highpass_period, + output=output_dir / "xvalidate_cmt.svg", + ) + + print(f"\nValidation complete. Plots saved to {output_dir}/") + + +if __name__ == "__main__": + main() From b8e683ee7306119bb23e8651caf87b062cdc516c Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 14:25:03 -0400 Subject: [PATCH 13/20] Updated the gitignore. --- EXAMPLES/green_function_database/regional/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EXAMPLES/green_function_database/regional/.gitignore b/EXAMPLES/green_function_database/regional/.gitignore index 38b8907fa..3c4d078ca 100644 --- a/EXAMPLES/green_function_database/regional/.gitignore +++ b/EXAMPLES/green_function_database/regional/.gitignore @@ -1,10 +1,12 @@ .snakemake +db_base/bin db_base/DATA/crust2.0 db_base/DATA/s362ani db_base/DATA/topo_bathy db_base/DATABASES_MPI db_base/OUTPUT_FILES db_base/mesher.flag +GFDB forward forward_cmt simulations From 094ed3ee255183c4992c9b8170b91675f540532f Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 14:55:54 -0400 Subject: [PATCH 14/20] Updated the workflow to allow for auto subsampling --- DATA/Par_file | 2 +- .../regional/db_base/DATA/Par_file | 2 +- src/shared/read_compute_parameters.f90 | 14 ++++++++++++++ src/shared/read_parameter_file.F90 | 4 ++-- src/shared/shared_par.f90 | 2 +- src/specfem3D/green_function_stf.F90 | 2 ++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/DATA/Par_file b/DATA/Par_file index 76344d82d..a258f63bd 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -444,7 +444,7 @@ HDF5_ENABLED = .true. # Green function database GF_DATABASE_ENABLED = .false. GF_DATABASE_PATH = OUTPUT_FILES/gf_database/ -GF_SUBSAMPLE_STEP = 4 +GF_SUBSAMPLE_STEP = 0 # 0 = auto (from mesh resolution), or set manually (>= 1) GF_BUFFER_SIZE = 100 GF_NEIGHBOR_SHELLS = 1 diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file index 5f9eeaaca..dee298d24 100644 --- a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file @@ -444,7 +444,7 @@ HDF5_ENABLED = .true. # Green function database GF_DATABASE_ENABLED = .true. GF_DATABASE_PATH = /Users/lsawade/specfem3d_globe/EXAMPLES/green_function_database/regional/GFDB/ -GF_SUBSAMPLE_STEP = 4 +GF_SUBSAMPLE_STEP = 0 # 0 = auto (from mesh resolution), or set manually (>= 1) GF_BUFFER_SIZE = 100 GF_NEIGHBOR_SHELLS = 1 DT = 0.1 \ No newline at end of file diff --git a/src/shared/read_compute_parameters.f90 b/src/shared/read_compute_parameters.f90 index c13908672..e64cb1578 100644 --- a/src/shared/read_compute_parameters.f90 +++ b/src/shared/read_compute_parameters.f90 @@ -198,6 +198,20 @@ subroutine rcp_set_compute_parameters() NSTEP = 100 * (int(RECORD_LENGTH_IN_MINUTES * 60.d0 / (100.d0*DT)) + 1) endif + ! Auto-compute GF_SUBSAMPLE_STEP if sentinel value 0 + if (GF_DATABASE_ENABLED .and. GF_SUBSAMPLE_STEP == 0) then + ! Nyquist constraint with 0.5x margin (2x oversampling): + ! step = floor(T_min_period / (4 * DT)) + GF_SUBSAMPLE_STEP = int(T_min_period / (4.0d0 * DT)) + if (GF_SUBSAMPLE_STEP < 1) GF_SUBSAMPLE_STEP = 1 + if (myrank == 0) then + print *, 'Green function database:' + print *, ' auto-computed GF_SUBSAMPLE_STEP = ', GF_SUBSAMPLE_STEP + print *, ' from T_min_period = ', sngl(T_min_period), ' DT = ', sngl(DT) + print * + endif + endif + ! steady state time step if (STEADY_STATE_KERNEL) then NSTEP_STEADY_STATE = nint(STEADY_STATE_LENGTH_IN_MINUTES * 60.d0 / DT) diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index 077d9af23..9f41ed728 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -466,8 +466,8 @@ subroutine read_parameter_file() ! checks GF database parameter validity if (GF_DATABASE_ENABLED) then - if (GF_SUBSAMPLE_STEP < 1) & - stop 'Error reading Par_file: GF_SUBSAMPLE_STEP must be >= 1' + if (GF_SUBSAMPLE_STEP < 0) & + stop 'Error reading Par_file: GF_SUBSAMPLE_STEP must be >= 0 (0 = auto)' if (GF_BUFFER_SIZE < 1) & stop 'Error reading Par_file: GF_BUFFER_SIZE must be >= 1' if (GF_NEIGHBOR_SHELLS < 0) & diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index 8f9968203..844ff8569 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -234,7 +234,7 @@ module shared_input_parameters ! Green function database logical :: GF_DATABASE_ENABLED = .false. character(len=MAX_STRING_LEN) :: GF_DATABASE_PATH = 'OUTPUT_FILES/gf_database/' - integer :: GF_SUBSAMPLE_STEP = 4 + integer :: GF_SUBSAMPLE_STEP = 0 ! 0 = auto-compute from T_min_period and DT integer :: GF_BUFFER_SIZE = 100 integer :: GF_NEIGHBOR_SHELLS = 1 logical :: GF_OVERWRITE = .false. diff --git a/src/specfem3D/green_function_stf.F90 b/src/specfem3D/green_function_stf.F90 index 8ed595d49..e432e488f 100644 --- a/src/specfem3D/green_function_stf.F90 +++ b/src/specfem3D/green_function_stf.F90 @@ -130,6 +130,8 @@ subroutine gf_compute_stf() write(IMAIN,*) ' lowpass cutoff frequency: ', sngl(gf_f_cutoff), ' Hz' write(IMAIN,*) ' filter order: ', GF_FILTER_ORDER write(IMAIN,*) ' subsample step: ', GF_SUBSAMPLE_STEP + write(IMAIN,*) ' subsampled Nyquist freq: ', sngl(gf_f_cutoff), ' Hz' + write(IMAIN,*) ' mesh max frequency: ', sngl(1.0d0/T_min_period), ' Hz' write(IMAIN,*) ' STF peak value: ', maxval(gf_stf) write(IMAIN,*) endif From a0a92ecea524b25c9d0a495514a56a547a685c57 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 15:30:50 -0400 Subject: [PATCH 15/20] Added small readme, and pyproj toml for `uv sync` --- EXAMPLES/green_function_database/README.md | 81 +++++++++++++++++++ .../green_function_database/pyproject.toml | 12 +++ 2 files changed, 93 insertions(+) create mode 100644 EXAMPLES/green_function_database/README.md create mode 100644 EXAMPLES/green_function_database/pyproject.toml diff --git a/EXAMPLES/green_function_database/README.md b/EXAMPLES/green_function_database/README.md new file mode 100644 index 000000000..7e625a674 --- /dev/null +++ b/EXAMPLES/green_function_database/README.md @@ -0,0 +1,81 @@ +# Green Function Database Example + +This example demonstrates how to build a Green function (GF) database using +SPECFEM3D_GLOBE and validate it against direct forward simulations. + +## Prerequisites + +- SPECFEM3D_GLOBE compiled with HDF5 and GF database support + (`GF_DATABASE_ENABLED = .true.` in `Par_file`) +- MPI (e.g., OpenMPI or MPICH) +- [uv](https://docs.astral.sh/uv/) for Python dependency management + +## Quick Start + +1. **Install Python dependencies** + + From this directory (`EXAMPLES/green_function_database/`): + + ```bash + uv sync + ``` + +2. **Run the regional workflow** + + ```bash + cd regional + snakemake -j3 + ``` + + This will: + - Set up the base directory and run the mesher + - Run reciprocal simulations (N, E, Z force components) for each station + - Build the GF database manifest (`GFDB/centroids.bin`) + - Run forward validation simulations (force and CMT) + - Produce cross-validation plots in `validation_output/` + +## Workflow Configuration + +The Snakefile accepts configuration overrides via `--config`. Key options: + +| Option | Default | Description | +|---------------------|---------------|------------------------------------------| +| `SPECFEM_DIR` | `../../..` | Path to the specfem3d_globe root | +| `NPROC` | `4` | Number of MPI ranks | +| `MPIRUN` | `mpirun` | MPI launcher command | +| `CREATE_VALIDATION` | `True` | Run validation forward simulations | + +Example with overrides: + +```bash +snakemake -j1 --config NPROC=6 MPIRUN="srun" +``` + +## Parallelism + +Stations can be run in parallel (components within a station are always +sequential). Use `-jN` with the `mpi` resource to control this: + +```bash +snakemake -j4 --resources mpi=2 +``` + +This allows up to 4 tasks in parallel, but limits MPI simulations to 2 +concurrent runs. + +## Cleaning Up + +```bash +cd regional +snakemake clean_all # remove all generated files +``` + +Or clean specific parts: + +```bash +snakemake clean_base # mesher output and symlinks +snakemake clean_simulations # station simulation directories +snakemake clean_database # GF database files +snakemake clean_validation # force validation directory +snakemake clean_validation_cmt # CMT validation directory +``` diff --git a/EXAMPLES/green_function_database/pyproject.toml b/EXAMPLES/green_function_database/pyproject.toml new file mode 100644 index 000000000..6db85dcee --- /dev/null +++ b/EXAMPLES/green_function_database/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "green-function-database" +version = "0.1.0" +description = "Green function database workflow for SPECFEM3D_GLOBE" +requires-python = ">=3.10" +dependencies = [ + "h5py", + "matplotlib", + "numpy", + "scipy", + "snakemake", +] From dcb3049a9d1676e1198b0a7eccad05f7ab9fae91 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 18:32:33 -0400 Subject: [PATCH 16/20] Implemented GPU transfer --- EXAMPLES/green_function_database/pyproject.toml | 1 + EXAMPLES/green_function_database/regional/Snakefile | 5 ++++- .../regional/db_base/DATA/Par_file | 10 +++++----- src/specfem3D/green_function_io.F90 | 11 +++++++++-- src/specfem3D/iterate_time.F90 | 2 ++ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/EXAMPLES/green_function_database/pyproject.toml b/EXAMPLES/green_function_database/pyproject.toml index 6db85dcee..0099edfdd 100644 --- a/EXAMPLES/green_function_database/pyproject.toml +++ b/EXAMPLES/green_function_database/pyproject.toml @@ -9,4 +9,5 @@ dependencies = [ "numpy", "scipy", "snakemake", + "obspy", ] diff --git a/EXAMPLES/green_function_database/regional/Snakefile b/EXAMPLES/green_function_database/regional/Snakefile index ed0e9b0fc..c70c49da4 100644 --- a/EXAMPLES/green_function_database/regional/Snakefile +++ b/EXAMPLES/green_function_database/regional/Snakefile @@ -345,7 +345,10 @@ rule build_manifest: manifest_csv=os.path.join(GF_DATABASE_PATH, "manifest.csv"), params: gf_db_path=GF_DATABASE_PATH, - script="/Users/lsawade/specfem3d_globe/utils/green_function/gf_build_manifest.py", + script=os.path.join( + os.path.abspath(SPECFEM_DIR), + "utils", "green_function", "gf_build_manifest.py", + ), shell: """ python {params.script} {params.gf_db_path} diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file index dee298d24..e8e02e434 100644 --- a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file @@ -419,12 +419,12 @@ BROADCAST_SAME_MESH_AND_MODEL = .false. #----------------------------------------------------------- # set to true to use GPUs -GPU_MODE = .false. +GPU_MODE = .true. # Only used if GPU_MODE = .true. : GPU_RUNTIME = 1 # 2 (OpenCL), 1 (Cuda) ou 0 (Compile-time -- does not work if configured with --with-cuda *AND* --with-opencl) GPU_PLATFORM = NVIDIA -GPU_DEVICE = Tesla +GPU_DEVICE = * # set to true to use the ADIOS library for I/Os ADIOS_ENABLED = .false. @@ -443,8 +443,8 @@ HDF5_ENABLED = .true. # Green function database GF_DATABASE_ENABLED = .true. -GF_DATABASE_PATH = /Users/lsawade/specfem3d_globe/EXAMPLES/green_function_database/regional/GFDB/ -GF_SUBSAMPLE_STEP = 0 # 0 = auto (from mesh resolution), or set manually (>= 1) +GF_DATABASE_PATH = /scratch/gpfs/TROMP/lsawade/specfem3d_globe/EXAMPLES/green_function_database/regional/GFDB/ +GF_SUBSAMPLE_STEP = 4 # 0 = auto (from mesh resolution), or set manually (>= 1) GF_BUFFER_SIZE = 100 GF_NEIGHBOR_SHELLS = 1 -DT = 0.1 \ No newline at end of file +DT = 0.1d0 \ No newline at end of file diff --git a/src/specfem3D/green_function_io.F90 b/src/specfem3D/green_function_io.F90 index 9faeca1c7..5d89a09c1 100644 --- a/src/specfem3D/green_function_io.F90 +++ b/src/specfem3D/green_function_io.F90 @@ -363,13 +363,14 @@ subroutine gf_write_timestep(it) #ifdef USE_HDF5 - use constants, only: CUSTOM_REAL,NGLLX,NGLLY,NGLLZ,GF_NCOMP_DISP + use constants, only: CUSTOM_REAL,NDIM,NGLLX,NGLLY,NGLLZ,GF_NCOMP_DISP use shared_parameters, only: GF_DATABASE_ENABLED,GF_SUBSAMPLE_STEP,GF_BUFFER_SIZE - use specfem_par, only: scale_displ + use specfem_par, only: scale_displ,GPU_MODE,Mesh_pointer,NGLOB_CRUST_MANTLE use specfem_par_crustmantle, only: & + displ_crust_mantle, & ibool => ibool_crust_mantle, & displ => displ_crust_mantle @@ -392,6 +393,12 @@ subroutine gf_write_timestep(it) ! check if this timestep is a subsampled output step if (mod(it, GF_SUBSAMPLE_STEP) /= 0) return + ! In GPU mode the wavefield lives on the device during the time loop; copy + ! the crust/mantle displacement back to the host before reading it. + if (GPU_MODE) then + call transfer_displ_cm_from_device(NDIM*NGLOB_CRUST_MANTLE, displ_crust_mantle, Mesh_pointer) + endif + ! increment buffer position gf_ibuf = gf_ibuf + 1 gf_isnap = gf_isnap + 1 diff --git a/src/specfem3D/iterate_time.F90 b/src/specfem3D/iterate_time.F90 index 24d514de8..066b8991f 100644 --- a/src/specfem3D/iterate_time.F90 +++ b/src/specfem3D/iterate_time.F90 @@ -218,6 +218,8 @@ subroutine iterate_time() ! write the seismograms with time shift (GPU_MODE transfer included) call write_seismograms() + + ! Green function database: buffer displacement at subsampled timesteps call gf_write_timestep(it) From 133bb967ca0446946ebc1aac3873390d2c8aa8cb Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 21:36:19 -0400 Subject: [PATCH 17/20] Fixed GPU and MPI issues --- src/specfem3D/compute_add_sources.f90 | 27 +++++---- src/specfem3D/green_function_morton.F90 | 59 ++++++++++--------- src/specfem3D/prepare_gpu.f90 | 1 + .../prepare_green_function_storage.f90 | 4 +- src/specfem3D/prepare_timerun.F90 | 5 ++ 5 files changed, 56 insertions(+), 40 deletions(-) diff --git a/src/specfem3D/compute_add_sources.f90 b/src/specfem3D/compute_add_sources.f90 index 1eb14ce03..6e5c93809 100644 --- a/src/specfem3D/compute_add_sources.f90 +++ b/src/specfem3D/compute_add_sources.f90 @@ -29,8 +29,6 @@ subroutine compute_add_sources() use specfem_par use specfem_par_crustmantle, only: accel_crust_mantle,ibool_crust_mantle - use shared_parameters, only: GF_DATABASE_ENABLED - use green_function_par, only: gf_stf implicit none @@ -78,14 +76,10 @@ subroutine compute_add_sources() timeval = time_t - tshift_src(isource) ! determines source time function value - if (GF_DATABASE_ENABLED) then - ! use precomputed Butterworth-filtered Gaussian STF - stf_used = gf_stf(it) - else - stf = get_stf_viscoelastic(timeval,isource,it) - ! distinguishes between single and double precision for reals - stf_used = real(stf,kind=CUSTOM_REAL) - endif + ! (the Green function database STF override is handled inside get_stf_viscoelastic) + stf = get_stf_viscoelastic(timeval,isource,it) + ! distinguishes between single and double precision for reals + stf_used = real(stf,kind=CUSTOM_REAL) ! adds source contribution do k = 1,NGLLZ @@ -498,6 +492,8 @@ double precision function get_stf_viscoelastic(time_source_dble,isource,it_index ! returns source time function value for specified time use specfem_par, only: USE_FORCE_POINT_SOURCE,USE_MONOCHROMATIC_CMT_SOURCE,force_stf,hdur,hdur_Gaussian,USE_SINSQ_STF + use shared_parameters, only: GF_DATABASE_ENABLED + use green_function_par, only: gf_stf implicit none @@ -513,6 +509,17 @@ double precision function get_stf_viscoelastic(time_source_dble,isource,it_index double precision, external :: comp_source_time_function_gauss_2 double precision, external :: comp_source_time_function_mono + ! Green function database: use the precomputed Butterworth-filtered Gaussian STF, + ! indexed by time step only (no source/tshift/stage dependence). Handling it here + ! mirrors the EXTERNAL_SOURCE_TIME_FUNCTION pattern so that both the CPU source + ! injection and the GPU local source-time-function array inherit it automatically. + ! GF database runs are SIMULATION_TYPE == 1 (forward) only, so it_index is the + ! forward time step. + if (GF_DATABASE_ENABLED) then + get_stf_viscoelastic = dble(gf_stf(it_index)) + return + endif + ! note: calling comp_source_time_function() includes the handling for external source time functions ! determines source time function value diff --git a/src/specfem3D/green_function_morton.F90 b/src/specfem3D/green_function_morton.F90 index b8008be24..b3ea5f326 100644 --- a/src/specfem3D/green_function_morton.F90 +++ b/src/specfem3D/green_function_morton.F90 @@ -60,35 +60,38 @@ subroutine gf_compute_morton_codes() ! local parameters integer :: i, ispec, iglob_center, ier - if (gf_nelem_local == 0) return - - ! allocate arrays - allocate(gf_center_xyz(3, gf_nelem_local), stat=ier) - if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_center_xyz') - - allocate(gf_morton_codes(gf_nelem_local), stat=ier) - if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_codes') - - allocate(gf_morton_hex(gf_nelem_local), stat=ier) - if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_hex') - - ! extract center GLL node coordinates and compute Morton codes - do i = 1, gf_nelem_local - ispec = gf_local_elements(i) - iglob_center = ibool(MIDX, MIDY, MIDZ, ispec) - - gf_center_xyz(1, i) = xstore(iglob_center) - gf_center_xyz(2, i) = ystore(iglob_center) - gf_center_xyz(3, i) = zstore(iglob_center) - - ! gf_morton_encode is a standalone function defined below - gf_morton_codes(i) = gf_morton_encode( & - gf_center_xyz(1, i), gf_center_xyz(2, i), gf_center_xyz(3, i)) - - write(gf_morton_hex(i), '(Z16.16)') gf_morton_codes(i) - enddo + ! NOTE: do not early-return on empty ranks. gf_report_morton_codes() below + ! contains a collective (gather_all_i) that every rank must reach, otherwise + ! ranks with no local GF elements would skip it and the run would deadlock. + if (gf_nelem_local > 0) then + ! allocate arrays + allocate(gf_center_xyz(3, gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_center_xyz') + + allocate(gf_morton_codes(gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_codes') + + allocate(gf_morton_hex(gf_nelem_local), stat=ier) + if (ier /= 0) call exit_MPI(myrank, 'Error allocating gf_morton_hex') + + ! extract center GLL node coordinates and compute Morton codes + do i = 1, gf_nelem_local + ispec = gf_local_elements(i) + iglob_center = ibool(MIDX, MIDY, MIDZ, ispec) + + gf_center_xyz(1, i) = xstore(iglob_center) + gf_center_xyz(2, i) = ystore(iglob_center) + gf_center_xyz(3, i) = zstore(iglob_center) + + ! gf_morton_encode is a standalone function defined below + gf_morton_codes(i) = gf_morton_encode( & + gf_center_xyz(1, i), gf_center_xyz(2, i), gf_center_xyz(3, i)) + + write(gf_morton_hex(i), '(Z16.16)') gf_morton_codes(i) + enddo + endif - ! report on rank 0 + ! report on rank 0 (collective: all ranks must call this) call gf_report_morton_codes() end subroutine gf_compute_morton_codes diff --git a/src/specfem3D/prepare_gpu.f90 b/src/specfem3D/prepare_gpu.f90 index dca38b28b..ce9135ad5 100644 --- a/src/specfem3D/prepare_gpu.f90 +++ b/src/specfem3D/prepare_gpu.f90 @@ -772,6 +772,7 @@ subroutine prepare_local_gpu_source_arrays() timeval = time_t - tshift_src(isource) ! source time function value (in range [-1,1] + ! (the Green function database STF override is handled inside get_stf_viscoelastic) stf = get_stf_viscoelastic(timeval,isource,it_tmp) ! distinguishes between single and double precision for reals diff --git a/src/specfem3D/prepare_green_function_storage.f90 b/src/specfem3D/prepare_green_function_storage.f90 index e8883c415..4417a2270 100644 --- a/src/specfem3D/prepare_green_function_storage.f90 +++ b/src/specfem3D/prepare_green_function_storage.f90 @@ -38,8 +38,8 @@ subroutine prepare_green_function_storage() if (.not. GF_DATABASE_ENABLED) return - ! precompute Butterworth-filtered STF - call gf_compute_stf() + ! note: the Butterworth-filtered STF (gf_stf) is precomputed earlier in + ! prepare_timerun(), before prepare_GPU() builds the local source arrays. ! compute Morton codes from element center coordinates call gf_compute_morton_codes() diff --git a/src/specfem3D/prepare_timerun.F90 b/src/specfem3D/prepare_timerun.F90 index 54f6764a6..7527c80f4 100644 --- a/src/specfem3D/prepare_timerun.F90 +++ b/src/specfem3D/prepare_timerun.F90 @@ -88,6 +88,11 @@ subroutine prepare_timerun() ! prepares oceans call prepare_oceans() + ! Green function database: precompute the Butterworth-filtered STF before + ! prepare_GPU(), because the GPU local source-time-function array is built + ! there and must use gf_stf (matching the CPU path in compute_add_sources()). + if (GF_DATABASE_ENABLED) call gf_compute_stf() + ! prepares GPU arrays call prepare_GPU() From b33f053a4aa5fdc809a8bf956224c87062ed0150 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Wed, 3 Jun 2026 21:46:57 -0400 Subject: [PATCH 18/20] cross validation interpolation more accurate traces do not match yet. --- utils/green_function/gf_cross_validate.py | 116 ++++++++++++++++------ utils/green_function/xvalidate.py | 10 +- 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/utils/green_function/gf_cross_validate.py b/utils/green_function/gf_cross_validate.py index 9a05c2642..e1b9a20ee 100644 --- a/utils/green_function/gf_cross_validate.py +++ b/utils/green_function/gf_cross_validate.py @@ -699,9 +699,71 @@ def read_forward_seismograms(fwd_dir, station): return traces, channel_prefix +def sinc_interp(y_coarse, time_coarse, time_fine): + """Whittaker-Shannon (finite-support sinc) interpolation onto a fine axis. + + Reconstructs the band-limited continuous signal from its uniform samples via + + recon(t) = sum_n y[n] * sinc((t - t_n) / Ts) + + where ``Ts`` is the coarse sampling interval. This is the ideal band-limited + interpolant. Because the sum runs only over the finite set of stored samples + (no circular wrap), it makes no periodicity assumption and is therefore safe + at the edges of a finite, non-periodic transient -- unlike FFT-based + ``resample``, whose periodic sinc wraps the endpoints and rings. It also has + a flat passband all the way to the band edge, unlike a cubic spline, which + droops over the upper part of the band. + + A non-zero endpoint level or tilt (e.g. the DC pedestal left by the + Gaussian->Heaviside cumsum in the CMT reconstruction) is a low-frequency + component that the truncated, slowly-decaying sinc kernel reproduces poorly + near the trace boundaries, producing a large spurious swing at the edges. To + avoid this, the straight line through the first and last samples is removed + before the sinc sum and added back (exactly, since a line is reproduced + exactly on the fine grid) afterward. This leaves an endpoint-zeroed, + well-behaved signal for the band-limited interpolation. + + Parameters + ---------- + y_coarse : ndarray, shape (n_coarse,) + Samples on the coarse, uniformly spaced grid. + time_coarse : ndarray, shape (n_coarse,) + Coarse time axis (uniform spacing ``Ts``). + time_fine : ndarray, shape (n_fine,) + Target (fine) time axis. + + Returns + ------- + ndarray, shape (n_fine,) + ``y_coarse`` reconstructed on ``time_fine``. + """ + # Remove the endpoint-connecting linear trend (kills the CMT DC pedestal and + # any tilt) before interpolating; restore it on the fine grid afterward. + slope = (y_coarse[-1] - y_coarse[0]) / (time_coarse[-1] - time_coarse[0]) + trend_coarse = y_coarse[0] + slope * (time_coarse - time_coarse[0]) + trend_fine = y_coarse[0] + slope * (time_fine - time_coarse[0]) + y_detrended = y_coarse - trend_coarse + + Ts = time_coarse[1] - time_coarse[0] + # M[i, n] = sinc((t_fine[i] - t_coarse[n]) / Ts); shape (n_fine, n_coarse). + # For the regional example this is ~18000 x 106, a small matmul. If a much + # finer/longer case makes M too large, chunk time_fine into blocks. + M = np.sinc((time_fine[:, None] - time_coarse[None, :]) / Ts) + return M @ y_detrended + trend_fine + + def align_and_filter(gf, time_gf, fwd_traces, dt, subsample_step, dt_sub, f_cutoff, lowpass_period, highpass_period): - """Lowpass-filter forward traces, subsample, align with GF, and apply optional filters. + """Lowpass-filter forward traces, upsample GF to forward rate, align, and filter. + + The comparison is carried out on the forward simulation's full-resolution + time axis. The forward traces are kept at their native sampling rate (only + anti-alias lowpassed to match the band-limited GF), and the subsampled GF + reconstruction is upsampled onto the forward axis with finite-support sinc + (Whittaker-Shannon) interpolation -- the ideal band-limited interpolant for + the finite, non-periodic GF transient. This gives an honest full-resolution + reconstruction error and avoids the edge ringing that FFT-based resampling + produces on a non-periodic signal. Parameters ---------- @@ -734,9 +796,11 @@ def align_and_filter(gf, time_gf, fwd_traces, dt, subsample_step, dt_sub, comps = list(fwd_traces.keys()) comp_labels = {comps[0]: "North", comps[1]: "East", comps[2]: "Vertical"} - # Lowpass filter forward traces to match GF anti-alias + # Lowpass filter forward traces to the GF band (kept at full resolution). + # This band-limits the forward trace to the same cutoff the GF database + # stored, so the comparison measures reconstruction error, not a band + # mismatch. lp_fc defaults to the database anti-alias cutoff. lp_fc = 1.0 / lowpass_period if lowpass_period else f_cutoff - lp_filter = lowpass_period is not None sos = butterworth_sos(4, lp_fc, 1.0 / dt) fwd_filtered = {} @@ -745,46 +809,34 @@ def align_and_filter(gf, time_gf, fwd_traces, dt, subsample_step, dt_sub, fwd_traces[comp]["amp"].astype(np.float64), sos ) - # Subsample forward traces - fwd_sub = {comp: fwd_filtered[comp][::subsample_step] for comp in comps} - time_fwd = fwd_traces[comps[0]]["time"][::subsample_step] + time_fwd = fwd_traces[comps[0]]["time"] - # Align traces using overlapping time window + # Align traces using the overlapping time window. The common axis is the + # forward (full-resolution) time axis. t_start = max(time_fwd[0], time_gf[0]) t_end = min(time_fwd[-1], time_gf[-1]) - mask_gf = (time_gf >= t_start) & (time_gf <= t_end) - time_common = time_gf[mask_gf] + mask_fwd = (time_fwd >= t_start) & (time_fwd <= t_end) + time_common = time_fwd[mask_fwd] + + fwd_aligned = {comp: fwd_filtered[comp][mask_fwd] for comp in comps} + # Upsample the GF reconstruction onto the forward axis with finite-support + # sinc interpolation (band-limited, non-periodic -> no edge ringing). gf_aligned = { - comps[0]: gf[0][mask_gf], - comps[1]: gf[1][mask_gf], - comps[2]: gf[2][mask_gf], + comps[i]: sinc_interp(gf[i], time_gf, time_common) + for i in range(3) } - fwd_aligned = {} + # Lowpass the reconstructed GF to the same band as the forward trace. The GF + # is already band-limited by construction, but this cleans any residual + # band-edge wiggle and keeps both traces in an identical band. for comp in comps: - fwd_aligned[comp] = np.interp(time_common, time_fwd, fwd_sub[comp]) - - # Taper both signals to suppress edge artifacts - n_compare = len(time_common) - n_taper = max(1, int(0.15 * n_compare)) - taper = np.ones(n_compare) - taper[:n_taper] = 0.5 * (1.0 - np.cos(np.linspace(0, pi, n_taper))) - taper[-n_taper:] = 0.5 * (1.0 - np.cos(np.linspace(pi, 0, n_taper))) - for comp in comps: - gf_aligned[comp] = gf_aligned[comp] * taper - fwd_aligned[comp] = fwd_aligned[comp] * taper - - # Optional lowpass filter on GF traces (when overriding database cutoff) - if lp_filter: - lp_sos_sub = butterworth_sos(4, 1.0 / lowpass_period, 1.0 / dt_sub) - for comp in comps: - gf_aligned[comp] = filter_with_taper(gf_aligned[comp], lp_sos_sub) + gf_aligned[comp] = filter_with_taper(gf_aligned[comp], sos) - # Optional highpass filter on both + # Optional highpass filter on both, at the forward sampling rate. if highpass_period is not None: - hp_sos = butterworth_highpass_sos(4, 1.0 / highpass_period, 1.0 / dt_sub) + hp_sos = butterworth_highpass_sos(4, 1.0 / highpass_period, 1.0 / dt) for comp in comps: gf_aligned[comp] = filter_with_taper(gf_aligned[comp], hp_sos) fwd_aligned[comp] = filter_with_taper(fwd_aligned[comp], hp_sos) diff --git a/utils/green_function/xvalidate.py b/utils/green_function/xvalidate.py index cb6ee0fab..0101ff09b 100644 --- a/utils/green_function/xvalidate.py +++ b/utils/green_function/xvalidate.py @@ -115,11 +115,11 @@ def main(): # Parse derived parameters station = parse_station_from_file(stations_file) - highpass_period = parse_min_period(values_from_mesher) + lowpass_period = parse_min_period(values_from_mesher) print(f"Base directory: {basedir}") print(f"Station: {station}") - print(f"Highpass period: {highpass_period:.1f} s (from mesh)") + print(f"Lowpass period: {lowpass_period:.1f} s (from mesh)") output_dir.mkdir(exist_ok=True) @@ -133,7 +133,8 @@ def main(): forward_output=forward_force_output, station=station, force=True, - highpass_period=highpass_period, + lowpass_period=lowpass_period, + highpass_period=0.004, output=output_dir / "xvalidate_force.svg", ) @@ -148,7 +149,8 @@ def main(): station=station, force=False, cmtsolution=cmtsolution, - highpass_period=highpass_period, + lowpass_period=lowpass_period, + highpass_period=0.004, output=output_dir / "xvalidate_cmt.svg", ) From 297e3a25afaf3bd0b2f61e29363ee61ebe550bf7 Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Thu, 4 Jun 2026 07:50:16 -0400 Subject: [PATCH 19/20] Added global Example --- EXAMPLES/green_function_database/README.md | 14 +- .../green_function_database/global/.gitignore | 13 + .../green_function_database/global/Snakefile | 604 ++++++++++++++++++ .../global/db_base/DATA/FORCESOLUTION | 11 + .../global/db_base/DATA/GF_LOCATIONS | 8 + .../global/db_base/DATA/Par_file | 450 +++++++++++++ .../global/db_base/DATA/STATIONS | 1 + .../global/validation_data/CMTSOLUTION | 13 + .../global/validation_data/FORCESOLUTION | 11 + .../global/validation_data/STATIONS | 18 + .../regional/db_base/DATA/Par_file | 4 +- src/shared/parallel.f90 | 35 +- src/shared/read_compute_parameters.f90 | 6 +- src/specfem3D/green_function_expand.F90 | 140 ++-- src/specfem3D/green_function_metadata.F90 | 15 +- utils/green_function/gf_cross_validate.py | 43 +- utils/green_function/xvalidate.py | 4 +- 17 files changed, 1284 insertions(+), 106 deletions(-) create mode 100644 EXAMPLES/green_function_database/global/.gitignore create mode 100644 EXAMPLES/green_function_database/global/Snakefile create mode 100644 EXAMPLES/green_function_database/global/db_base/DATA/FORCESOLUTION create mode 100644 EXAMPLES/green_function_database/global/db_base/DATA/GF_LOCATIONS create mode 100644 EXAMPLES/green_function_database/global/db_base/DATA/Par_file create mode 100644 EXAMPLES/green_function_database/global/db_base/DATA/STATIONS create mode 100644 EXAMPLES/green_function_database/global/validation_data/CMTSOLUTION create mode 100644 EXAMPLES/green_function_database/global/validation_data/FORCESOLUTION create mode 100644 EXAMPLES/green_function_database/global/validation_data/STATIONS diff --git a/EXAMPLES/green_function_database/README.md b/EXAMPLES/green_function_database/README.md index 7e625a674..04ab41da9 100644 --- a/EXAMPLES/green_function_database/README.md +++ b/EXAMPLES/green_function_database/README.md @@ -24,7 +24,7 @@ SPECFEM3D_GLOBE and validate it against direct forward simulations. ```bash cd regional - snakemake -j3 + snakemake -j1 ``` This will: @@ -34,6 +34,18 @@ SPECFEM3D_GLOBE and validate it against direct forward simulations. - Run forward validation simulations (force and CMT) - Produce cross-validation plots in `validation_output/` + **Note 1**: The regional workflow is affected by the absorbing boundary conditions. It is important to choose the stations carefully to avoid strong reflections from the boundaries. + +2. **Run the global workflow** + + ```bash + cd global + snakemake -j1 + ``` + + **Note 1**: The global workflow is computationally more expensive but does not have issues with boundary reflections. It is recommended to run the global workflow if you have sufficient computational resources. + + ## Workflow Configuration The Snakefile accepts configuration overrides via `--config`. Key options: diff --git a/EXAMPLES/green_function_database/global/.gitignore b/EXAMPLES/green_function_database/global/.gitignore new file mode 100644 index 000000000..3c4d078ca --- /dev/null +++ b/EXAMPLES/green_function_database/global/.gitignore @@ -0,0 +1,13 @@ +.snakemake +db_base/bin +db_base/DATA/crust2.0 +db_base/DATA/s362ani +db_base/DATA/topo_bathy +db_base/DATABASES_MPI +db_base/OUTPUT_FILES +db_base/mesher.flag +GFDB +forward +forward_cmt +simulations +validation_output \ No newline at end of file diff --git a/EXAMPLES/green_function_database/global/Snakefile b/EXAMPLES/green_function_database/global/Snakefile new file mode 100644 index 000000000..586c6db58 --- /dev/null +++ b/EXAMPLES/green_function_database/global/Snakefile @@ -0,0 +1,604 @@ +""" +Snakemake workflow for Green Function database generation. + +For each station in the STATIONS file, runs 3 reciprocal simulations (N, E, Z force +components) sequentially from the same directory, then builds the manifest. + +All simulations write to a shared GF_DATABASE_PATH (absolute path from Par_file). +The 3 components per station are sequential so each writes its force-component slice +into the same HDF5 files without conflict. + +Usage: + snakemake -j1 # fully sequential + snakemake -jN --resources mpi=1 # N stations in parallel (components sequential) + +Configuration (edit below or override via --config): + SPECFEM_DIR: path to specfem3d_globe repository root (default: ../../..) + BASEDIR: path to base simulation template (contains DATA/Par_file, DATA/STATIONS) + STATIONS: path to STATIONS file + NPROC: number of MPI ranks + MPIRUN: MPI launcher command + CREATE_VALIDATION: True/False — create a forward simulation directory for validation + VALIDATION_DATA: path to validation data (STATIONS, CMTSOLUTION, FORCESOLUTION) + VALIDATION_DIR: output directory for the validation forward simulation +""" + +import os +import shutil + +# ─── Configuration ─────────────────────────────────────────────────────────── + +# Path to the specfem3d_globe repository root (for binaries and DATA files) +SPECFEM_DIR = config.get("SPECFEM_DIR", os.path.join("..", "..", "..")) + +# Base directory containing the meshed model (bin/, DATABASES_MPI/, DATA/Par_file) +BASEDIR = config.get("BASEDIR", "db_base") + +# Stations file +STATIONS_FILE = config.get("STATIONS", os.path.join(BASEDIR, "DATA", "STATIONS")) + +# MPI configuration +NPROC = int(config.get("NPROC", 24)) +MPIRUN = config.get("MPIRUN", "mpirun") + +# Force source parameters +# Note: f0 is NOT used for GF STF computation (gf_hdur is derived from T_min_period). +# It only affects the initial source half-duration for time-step stability checks. +F0 = config.get("F0", "0.0500") +FACTOR_FORCE = config.get("FACTOR_FORCE", "1.0d15") + +# Output root for all station simulations +SIMDIR = config.get("SIMDIR", "simulations") + +# Validation forward simulation toggle +CREATE_VALIDATION = config.get("CREATE_VALIDATION", True) +VALIDATION_DATA = config.get("VALIDATION_DATA", "validation_data") +VALIDATION_DIR = config.get("VALIDATION_DIR", "forward") +VALIDATION_CMT_DIR = config.get("VALIDATION_CMT_DIR", "forward_cmt") + + +# ─── GF database output path ───────────────────────────────────────────────── + +# Default: GFDB/ inside the example directory (resolved to absolute path) +GF_DATABASE_PATH = os.path.abspath(config.get("GF_DATABASE_PATH", "GFDB")) + + +# ─── Parse STATIONS file ───────────────────────────────────────────────────── + +def parse_stations(stations_file): + """Parse SPECFEM STATIONS file.""" + stations = [] + with open(stations_file) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) >= 6: + stations.append({ + "station": parts[0], + "network": parts[1], + "latitude": parts[2], + "longitude": parts[3], + "elevation": parts[4], + "burial": parts[5], + }) + return stations + + +STATIONS = parse_stations(STATIONS_FILE) +STATION_IDS = [f"{s['network']}.{s['station']}" for s in STATIONS] +COMPONENTS = ["N", "E", "Z"] + + +# ─── Helper functions ──────────────────────────────────────────────────────── + +def get_station_info(station_id): + """Look up station metadata by network.station ID.""" + for s in STATIONS: + if f"{s['network']}.{s['station']}" == station_id: + return s + raise ValueError(f"Station {station_id} not found") + + +def forcesolution_content(station_id, comp): + """Generate FORCESOLUTION file content for a given station and component.""" + s = get_station_info(station_id) + comp_n = "1.0" if comp == "N" else "0.0" + comp_e = "1.0" if comp == "E" else "0.0" + comp_z = "1.0" if comp == "Z" else "0.0" + + return ( + f"FORCE {station_id}\n" + f"time shift: 0.0000\n" + f"f0: {F0}\n" + f"latitude: {s['latitude']}\n" + f"longitude: {s['longitude']}\n" + f"depth: {float(s['burial']):.4f}\n" + f"source time function: 0\n" + f"factor force source: {FACTOR_FORCE}\n" + f"comp dir vect source E: {comp_e}\n" + f"comp dir vect source N: {comp_n}\n" + f"comp dir vect source Z: {comp_z}\n" + ) + + +# ─── Rules ─────────────────────────────────────────────────────────────────── + +def all_inputs(wildcards): + """Collect all target files for the workflow.""" + inputs = [] + # All Z simulations done (Z runs last, so implies N and E are done too) + inputs += expand(os.path.join(SIMDIR, "{station}", "done_Z.flag"), + station=STATION_IDS) + # Manifest built + inputs.append(os.path.join(GF_DATABASE_PATH, "centroids.bin")) + # Optionally run validation forward simulations and cross-validation plots + if CREATE_VALIDATION: + inputs.append(os.path.join("validation_output", "xvalidate_force.svg")) + inputs.append(os.path.join("validation_output", "xvalidate_cmt.svg")) + return inputs + + +rule all: + input: + all_inputs, + + +rule setup_base: + """Set up the base directory: link specfem binaries and data, then run the mesher.""" + output: + flag=os.path.join(BASEDIR, "mesher.flag"), + params: + basedir=BASEDIR, + specfem_dir=SPECFEM_DIR, + gf_database_path=GF_DATABASE_PATH, + nproc=NPROC, + mpirun=MPIRUN, + run: + import re + + basedir = params.basedir + specfem_dir = os.path.abspath(params.specfem_dir) + + # Create required directories + os.makedirs(os.path.join(basedir, "DATABASES_MPI"), exist_ok=True) + os.makedirs(os.path.join(basedir, "OUTPUT_FILES"), exist_ok=True) + os.makedirs(params.gf_database_path, exist_ok=True) + + # Symlink bin/ to specfem binaries + bin_link = os.path.join(basedir, "bin") + bin_target = os.path.join(specfem_dir, "bin") + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + + # Symlink required DATA subdirectories + for name in ["topo_bathy", "s362ani", "crust2.0"]: + link = os.path.join(basedir, "DATA", name) + target = os.path.join(specfem_dir, "DATA", name) + if not os.path.exists(link): + os.symlink(target, link) + + # Update GF_DATABASE_PATH in Par_file to match the configured path + parfile = os.path.join(basedir, "DATA", "Par_file") + with open(parfile, "r") as f: + content = f.read() + content = re.sub( + r"^(GF_DATABASE_PATH\s*=\s*).*$", + rf"\g<1>{params.gf_database_path}/", + content, + flags=re.MULTILINE, + ) + with open(parfile, "w") as f: + f.write(content) + + # Run the mesher + shell( + "cd {basedir} && " + "{params.mpirun} -np {params.nproc} ./bin/xmeshfem3D " + "> OUTPUT_FILES/output_mesher.txt 2>&1" + ) + + # Touch flag + with open(output.flag, "w") as f: + f.write("mesher complete\n") + + +rule setup_station: + """Create simulation directory for a station (shared by all 3 components).""" + input: + base=os.path.join(BASEDIR, "mesher.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "setup.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + basedir=BASEDIR, + stations_file=STATIONS_FILE, + run: + simdir = params.simdir + + # Create directories + os.makedirs(os.path.join(simdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(simdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ + bin_link = os.path.join(simdir, "bin") + db_link = os.path.join(simdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file + shutil.copy2( + os.path.join(params.basedir, "DATA", "Par_file"), + os.path.join(simdir, "DATA", "Par_file"), + ) + + # Copy STATIONS file + shutil.copy2(params.stations_file, os.path.join(simdir, "DATA", "STATIONS")) + + # Copy GF_LOCATIONS if it exists + gf_loc_src = os.path.join(params.basedir, "DATA", "GF_LOCATIONS") + if os.path.exists(gf_loc_src): + shutil.copy2(gf_loc_src, os.path.join(simdir, "DATA", "GF_LOCATIONS")) + + # Touch flag + with open(output.flag, "w") as f: + f.write("setup complete\n") + + +rule run_component_N: + """Run N-component simulation (first).""" + input: + setup=os.path.join(SIMDIR, "{station}", "setup.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_N.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for N component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "N")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_N.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("N complete\n") + + +rule run_component_E: + """Run E-component simulation (after N).""" + input: + prev=os.path.join(SIMDIR, "{station}", "done_N.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_E.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for E component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "E")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_E.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("E complete\n") + + +rule run_component_Z: + """Run Z-component simulation (after E).""" + input: + prev=os.path.join(SIMDIR, "{station}", "done_E.flag"), + output: + flag=os.path.join(SIMDIR, "{station}", "done_Z.flag"), + params: + simdir=lambda wc: os.path.join(SIMDIR, wc.station), + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + simdir = params.simdir + # Write FORCESOLUTION for Z component + with open(os.path.join(simdir, "DATA", "FORCESOLUTION"), "w") as f: + f.write(forcesolution_content(wildcards.station, "Z")) + # Run solver + shell( + "cd {simdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver_Z.txt 2>&1" + ) + # Touch flag + with open(output.flag, "w") as f: + f.write("Z complete\n") + + +rule build_manifest: + """Build centroids.bin and manifest.csv after all stations complete.""" + input: + expand(os.path.join(SIMDIR, "{station}", "done_Z.flag"), + station=STATION_IDS), + output: + centroids=os.path.join(GF_DATABASE_PATH, "centroids.bin"), + manifest_csv=os.path.join(GF_DATABASE_PATH, "manifest.csv"), + params: + gf_db_path=GF_DATABASE_PATH, + script=os.path.join( + os.path.abspath(SPECFEM_DIR), + "utils", "green_function", "gf_build_manifest.py", + ), + shell: + """ + python {params.script} {params.gf_db_path} + """ + +rule setup_validation_forward: + """Set up a forward simulation directory for validation using data from validation_data/.""" + input: + # Only run after the GF database is built + os.path.join(GF_DATABASE_PATH, "centroids.bin"), + output: + flag=os.path.join(VALIDATION_DIR, "setup.flag"), + params: + basedir=BASEDIR, + validation_data=VALIDATION_DATA, + validation_dir=VALIDATION_DIR, + run: + import re + + vdir = params.validation_dir + + # Create directories + os.makedirs(os.path.join(vdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(vdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ from base directory + bin_link = os.path.join(vdir, "bin") + db_link = os.path.join(vdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file from base directory and disable GF_DATABASE_ENABLED + parfile_src = os.path.join(params.basedir, "DATA", "Par_file") + parfile_dst = os.path.join(vdir, "DATA", "Par_file") + with open(parfile_src, "r") as f: + parfile_content = f.read() + parfile_content = re.sub( + r"^(\s*GF_DATABASE_ENABLED\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + # Ensure force source is enabled for force validation + parfile_content = re.sub( + r"^(USE_FORCE_POINT_SOURCE\s*=\s*)\.false\.", + r"\1.true.", + parfile_content, + flags=re.MULTILINE, + ) + with open(parfile_dst, "w") as f: + f.write(parfile_content) + + # Copy STATIONS from validation_data + shutil.copy2( + os.path.join(params.validation_data, "STATIONS"), + os.path.join(vdir, "DATA", "STATIONS"), + ) + + # Copy CMTSOLUTION if it exists + cmt_src = os.path.join(params.validation_data, "CMTSOLUTION") + if os.path.exists(cmt_src): + shutil.copy2(cmt_src, os.path.join(vdir, "DATA", "CMTSOLUTION")) + + # Copy FORCESOLUTION if it exists + force_src = os.path.join(params.validation_data, "FORCESOLUTION") + if os.path.exists(force_src): + shutil.copy2(force_src, os.path.join(vdir, "DATA", "FORCESOLUTION")) + + # Touch flag + with open(output.flag, "w") as f: + f.write("validation forward setup complete\n") + + +rule run_validation_forward: + """Run the validation forward simulation.""" + input: + setup=os.path.join(VALIDATION_DIR, "setup.flag"), + output: + flag=os.path.join(VALIDATION_DIR, "done.flag"), + params: + validation_dir=VALIDATION_DIR, + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + vdir = params.validation_dir + shell( + "cd {vdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver.txt 2>&1" + ) + with open(output.flag, "w") as f: + f.write("validation forward complete\n") + + +rule setup_validation_cmt: + """Set up a CMT forward simulation directory for validation.""" + input: + # Only run after the GF database is built + os.path.join(GF_DATABASE_PATH, "centroids.bin"), + output: + flag=os.path.join(VALIDATION_CMT_DIR, "setup.flag"), + params: + basedir=BASEDIR, + validation_data=VALIDATION_DATA, + validation_cmt_dir=VALIDATION_CMT_DIR, + run: + import re + + vdir = params.validation_cmt_dir + + # Create directories + os.makedirs(os.path.join(vdir, "DATA"), exist_ok=True) + os.makedirs(os.path.join(vdir, "OUTPUT_FILES"), exist_ok=True) + + # Symlink bin/ and DATABASES_MPI/ from base directory + bin_link = os.path.join(vdir, "bin") + db_link = os.path.join(vdir, "DATABASES_MPI") + bin_target = os.path.abspath(os.path.join(params.basedir, "bin")) + db_target = os.path.abspath(os.path.join(params.basedir, "DATABASES_MPI")) + + if not os.path.exists(bin_link): + os.symlink(bin_target, bin_link) + if not os.path.exists(db_link): + os.symlink(db_target, db_link) + + # Copy Par_file from base directory, disable GF and set CMT source + parfile_src = os.path.join(params.basedir, "DATA", "Par_file") + parfile_dst = os.path.join(vdir, "DATA", "Par_file") + with open(parfile_src, "r") as f: + parfile_content = f.read() + parfile_content = re.sub( + r"^(\s*GF_DATABASE_ENABLED\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + # Disable force point source so SPECFEM uses CMTSOLUTION + parfile_content = re.sub( + r"^(USE_FORCE_POINT_SOURCE\s*=\s*)\.true\.", + r"\1.false.", + parfile_content, + flags=re.MULTILINE, + ) + with open(parfile_dst, "w") as f: + f.write(parfile_content) + + # Copy STATIONS from validation_data + shutil.copy2( + os.path.join(params.validation_data, "STATIONS"), + os.path.join(vdir, "DATA", "STATIONS"), + ) + + # Copy CMTSOLUTION (required for CMT simulation) + cmt_src = os.path.join(params.validation_data, "CMTSOLUTION") + if os.path.exists(cmt_src): + shutil.copy2(cmt_src, os.path.join(vdir, "DATA", "CMTSOLUTION")) + else: + raise FileNotFoundError(f"CMTSOLUTION not found at {cmt_src}") + + # Touch flag + with open(output.flag, "w") as f: + f.write("validation CMT setup complete\n") + + +rule run_validation_cmt: + """Run the CMT validation forward simulation.""" + input: + setup=os.path.join(VALIDATION_CMT_DIR, "setup.flag"), + output: + flag=os.path.join(VALIDATION_CMT_DIR, "done.flag"), + params: + validation_cmt_dir=VALIDATION_CMT_DIR, + nproc=NPROC, + mpirun=MPIRUN, + resources: + mpi=1, + run: + vdir = params.validation_cmt_dir + shell( + "cd {vdir} && " + "{params.mpirun} -np {params.nproc} ./bin/xspecfem3D " + "> OUTPUT_FILES/output_solver.txt 2>&1" + ) + with open(output.flag, "w") as f: + f.write("validation CMT forward complete\n") + + +rule run_cross_validation: + """Run cross-validation comparing GF database against forward simulations.""" + input: + force_done=os.path.join(VALIDATION_DIR, "done.flag"), + cmt_done=os.path.join(VALIDATION_CMT_DIR, "done.flag"), + output: + force_svg=os.path.join("validation_output", "xvalidate_force.svg"), + cmt_svg=os.path.join("validation_output", "xvalidate_cmt.svg"), + params: + script=os.path.join( + os.path.abspath(SPECFEM_DIR), + "utils", "green_function", "xvalidate.py", + ), + shell: + """ + python {params.script} . + """ + + +rule clean_base: + """Clean base directory links and mesher output.""" + shell: + "rm -rf {BASEDIR}/bin {BASEDIR}/DATABASES_MPI {BASEDIR}/OUTPUT_FILES " + "{BASEDIR}/DATA/topo_bathy {BASEDIR}/DATA/s362ani {BASEDIR}/DATA/crust2.0 " + "{BASEDIR}/mesher.flag" + +rule clean_simulations: + """Clean all simulation directories and output files.""" + shell: + "rm -rf {SIMDIR}/*" + +rule clean_validation: + """Clean validation forward simulation directory.""" + shell: + "rm -rf {VALIDATION_DIR}/*" + +rule clean_validation_cmt: + """Clean CMT validation forward simulation directory.""" + shell: + "rm -rf {VALIDATION_CMT_DIR}/*" + +rule clean_cross_validation: + """Clean cross-validation output files.""" + shell: + "rm -rf validation_output/" + +rule clean_database: + """Clean GF database output files.""" + shell: + "rm -rf {GF_DATABASE_PATH}/*" + +rule clean_all: + """Clean base, simulations, validation, and database outputs.""" + shell: + "rm -rf {BASEDIR}/bin {BASEDIR}/DATABASES_MPI {BASEDIR}/OUTPUT_FILES " + "{BASEDIR}/DATA/topo_bathy {BASEDIR}/DATA/s362ani {BASEDIR}/DATA/crust2.0 " + "{BASEDIR}/mesher.flag " + "{SIMDIR}/* {GF_DATABASE_PATH}/* {VALIDATION_DIR}/* {VALIDATION_CMT_DIR}/* " + "validation_output/*" \ No newline at end of file diff --git a/EXAMPLES/green_function_database/global/db_base/DATA/FORCESOLUTION b/EXAMPLES/green_function_database/global/db_base/DATA/FORCESOLUTION new file mode 100644 index 000000000..95047dab5 --- /dev/null +++ b/EXAMPLES/green_function_database/global/db_base/DATA/FORCESOLUTION @@ -0,0 +1,11 @@ +FORCE IU.SJG +time shift: 0.0000 +f0: 0.0500 +latitude: -13.8200 +longitude: -67.2500 +depth: 647.1000 +source time function: 0 +factor force source: 1.0d15 +comp dir vect source E: 0.0 +comp dir vect source N: 0.0 +comp dir vect source Z: 1.0 diff --git a/EXAMPLES/green_function_database/global/db_base/DATA/GF_LOCATIONS b/EXAMPLES/green_function_database/global/db_base/DATA/GF_LOCATIONS new file mode 100644 index 000000000..c1ea83fac --- /dev/null +++ b/EXAMPLES/green_function_database/global/db_base/DATA/GF_LOCATIONS @@ -0,0 +1,8 @@ +# GF_LOCATIONS - earthquake hypocenter locations for Green function database +# latitude longitude depth_km +# 2010 Chile earthquake (Maule) +-35.909 -72.733 35.0 +# 2019 Peru earthquake +-5.812 -75.270 122.6 +# Near center of chunk +-13.0 -67.0 10.0 diff --git a/EXAMPLES/green_function_database/global/db_base/DATA/Par_file b/EXAMPLES/green_function_database/global/db_base/DATA/Par_file new file mode 100644 index 000000000..4348b426f --- /dev/null +++ b/EXAMPLES/green_function_database/global/db_base/DATA/Par_file @@ -0,0 +1,450 @@ +#----------------------------------------------------------- +# +# Simulation input parameters +# +#----------------------------------------------------------- + +# forward or adjoint simulation +SIMULATION_TYPE = 1 # set to 1 for forward simulations, 2 for adjoint simulations for sources, and 3 for kernel simulations +NOISE_TOMOGRAPHY = 0 # flag of noise tomography, three steps (1,2,3). If earthquake simulation, set it to 0. +SAVE_FORWARD = .false. # save last frame of forward simulation or not + +# number of chunks (1,2,3 or 6) +NCHUNKS = 6 + +# angular width of the first chunk (not used if full sphere with six chunks) +ANGULAR_WIDTH_XI_IN_DEGREES = 90.d0 # angular size of a chunk +ANGULAR_WIDTH_ETA_IN_DEGREES = 90.d0 +CENTER_LATITUDE_IN_DEGREES = -13.8200d0 +CENTER_LONGITUDE_IN_DEGREES = -67.2500 +GAMMA_ROTATION_AZIMUTH = 0.d0 + +# number of elements at the surface along the two sides of the first chunk +# (must be multiple of 16 and 8 * multiple of NPROC below) +NEX_XI = 64 +NEX_ETA = 64 + +# number of MPI processors along the two sides of the first chunk +NPROC_XI = 2 +NPROC_ETA = 2 + +#----------------------------------------------------------- +# +# Model +# +#----------------------------------------------------------- + +# 1D models with real structure: +# 1D_isotropic_prem, 1D_transversely_isotropic_prem, 1D_iasp91, 1D_1066a, 1D_ak135f_no_mud, 1D_ref, 1D_ref_iso, 1D_jp3d, 1D_sea99, 1D_Berkeley +# +# 1D models with only one fictitious averaged crustal layer: +# 1D_isotropic_prem_onecrust, 1D_transversely_isotropic_prem_onecrust, 1D_iasp91_onecrust, 1D_1066a_onecrust, 1D_ak135f_no_mud_onecrust +# +# fully 3D models: +# transversely_isotropic_prem_plus_3D_crust_2.0, 3D_anisotropic, 3D_attenuation, +# s20rts, s40rts, s362ani, s362iso, s362wmani, s362ani_prem, s362ani_3DQ, s362iso_3DQ, +# s29ea, sea99_jp3d1994, sea99, jp3d1994, heterogen, full_sh, sgloberani_aniso, sgloberani_iso, +# spiral, emc_model, emc_model_qmu, emc_model_tiso, emc_model_tiso_qmu, semucb_A3d, semucb_A3d_3dQ +# (see manual for more...) +# +# 3D crustal models: +# crust1.0, crust2.0, EPcrust, EuCRUST, crustmaps, crustSH +# +# Mars models: +# 1D_Sohl, 1D_Sohl_3D_crust, 1D_case65TAY, 1D_case65TAY_3D_crust, mars_1D, mars_1D_3D_crust +# +# Moon models: +# vpremoon +# +# 3D models with 3D crust: append "_**crustname**" to the mantle model name +# to take a 3D crustal model (by default crust2.0 is taken for 3D mantle models) +# e.g. s20rts_crust1.0, s362ani_crustmaps, full_sh_crustSH, sglobe_EPcrust, etc. +# +# 3D models with 1D crust: append "_1Dcrust" to the 3D model name +# to take the 1D crustal model from the +# associated reference model rather than the default 3D crustal model +# e.g. s20rts_1Dcrust, s362ani_1Dcrust, etc. +# +MODEL = s362ani + +# parameters describing the Earth model +OCEANS = .true. +ELLIPTICITY = .true. +TOPOGRAPHY = .true. +GRAVITY = .true. +ROTATION = .true. +ATTENUATION = .true. + +# full gravity calculation by solving Poisson's equation for gravity potential instead of using a Cowling approximation +# (must have also GRAVITY flag set to .true. to become active) +FULL_GRAVITY = .false. +# for full gravity calculation, set to 0 == builtin or 1 == PETSc Poisson solver +# (the PETSc solver option needs the PETSc library installed; code configuration --with-petsc) +POISSON_SOLVER = 0 + +# record length in minutes +RECORD_LENGTH_IN_MINUTES = 30.0d0 + +#----------------------------------------------------------- +# +# Mesh +# +#----------------------------------------------------------- + +## regional mesh cut-off +# using this flag will cut-off the mesh in the mantle at a layer matching to the given cut-off depth. +# this flag only has an effect for regional simulations, i.e., for NCHUNKS values less than 6. +REGIONAL_MESH_CUTOFF = .false. + +# regional mesh cut-off depth (in km) +# possible selections are: 24.4d0, 80.d0, 220.d0, 400.d0, 600.d0, 670.d0, 771.d0 +REGIONAL_MESH_CUTOFF_DEPTH = 771.d0 + +# regional mesh cut-off w/ a second doubling layer below 220km interface +# (by default, a first doubling layer will be added below the Moho, and a second one below the 771km-depth layer. +# Setting this flag to .true., will move the second one below the 220km-depth layer for regional mesh cut-offs only.) +REGIONAL_MESH_ADD_2ND_DOUBLING = .false. + +#----------------------------------------------------------- +# +# Absorbing boundary conditions +# +#----------------------------------------------------------- + +# absorbing boundary conditions for a regional simulation +ABSORBING_CONDITIONS = .false. + +# run global simulation for a circular region and apply high attenuation for the rest of the model +# this creates an absorbing boundary with less reflection than stacey at a cost of 6x the computational cost +# NCHUNKS must be set to 6 to enable this flag +ABSORB_USING_GLOBAL_SPONGE = .false. + +# location and size of the region with no sponge (the region to run simulation) +SPONGE_LATITUDE_IN_DEGREES = 40.d0 +SPONGE_LONGITUDE_IN_DEGREES = 25.d0 +SPONGE_RADIUS_IN_DEGREES = 25.d0 + +#----------------------------------------------------------- +# +# undoing attenuation for sensitivity kernel calculations +# +#----------------------------------------------------------- + +# to undo attenuation for sensitivity kernel calculations or forward runs with SAVE_FORWARD +# use one (and only one) of the two flags below. UNDO_ATTENUATION is much better (it is exact) +# but requires a significant amount of disk space for temporary storage. +PARTIAL_PHYS_DISPERSION_ONLY = .false. +UNDO_ATTENUATION = .false. + +## undo attenuation memory +# How much memory (in GB) is installed on your machine per CPU core +# (only used for UNDO_ATTENUATION, can be ignored otherwise) +# Beware, this value MUST be given per core, i.e. per MPI thread, i.e. per MPI rank, NOT per node. +# This value is for instance: +# - 4 GB on Tiger at Princeton +# - 4 GB on TGCC Curie in Paris +# - 4 GB on Titan at ORNL when using CPUs only (no GPUs); start your run with "aprun -n$NPROC -N8 -S4 -j1" +# - 2 GB on the machine used by Christina Morency +# - 2 GB on the TACC machine used by Min Chen +# - 1.5 GB on the GPU cluster in Marseille +# When running on GPU machines, it is simpler to set PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0 +# and then set MEMORY_INSTALLED_PER_CORE_IN_GB to the amount of memory that you estimate is free (rather than installed) +# on the host of the GPU card while running your GPU job. +# For GPU runs on Titan at ORNL, use PERCENT_OF_MEM_TO_USE_PER_CORE = 100.d0 and MEMORY_INSTALLED_PER_CORE_IN_GB = 25.d0 +# and run your job with "aprun -n$NPROC -N1 -S1 -j1" +# (each host has 32 GB on Titan, each GPU has 6 GB, thus even if all the GPU arrays are duplicated on the host +# this leaves 32 - 6 = 26 GB free on the host; leaving 1 GB for the Linux system, we can safely use 100% of 25 GB) +MEMORY_INSTALLED_PER_CORE_IN_GB = 4.d0 +# What percentage of this total do you allow us to use for arrays to undo attenuation, keeping in mind that you +# need to leave some memory available for the GNU/Linux system to run +# (a typical value is 85%; any value below is fine but the code will then save a lot of data to disk; +# values above, say 90% or 92%, can be OK on some systems but can make the adjoint code run out of memory +# on other systems, depending on how much memory per node the GNU/Linux system needs for itself; thus you can try +# a higher value and if the adjoint crashes then try again with a lower value) +PERCENT_OF_MEM_TO_USE_PER_CORE = 85.d0 + +## exact mass matrices for rotation +# three mass matrices instead of one are needed to handle rotation very accurately; +# otherwise rotation is handled slightly less accurately (but still reasonably well); +# set to .true. if you are interested in precise effects related to rotation; +# set to .false. if you are solving very large inverse problems at high frequency and also undoing attenuation exactly +# +# using the UNDO_ATTENUATION flag above, in which case saving as much memory as possible can be a good idea. +# You can also safely set it to .false. if you are not in a period range in which rotation matters, +# e.g. if you are targetting very short-period body waves. if in doubt, set to .true. +# +# You can safeely set it to .true. if you have ABSORBING_CONDITIONS above, because in that case the code +# will use three mass matrices anyway and thus there is no additional memory cost. +# this flag is of course unused if ROTATION above is set to .false. +EXACT_MASS_MATRIX_FOR_ROTATION = .true. + +#----------------------------------------------------------- +# +# LDDRK time scheme +# +#----------------------------------------------------------- + +# this for LDDRK high-order time scheme instead of Newmark +USE_LDDRK = .false. + +# the maximum CFL of LDDRK is significantly higher than that of the Newmark scheme, +# in a ratio that is theoretically 1.327 / 0.697 = 1.15 / 0.604 = 1.903 for a solid with Poisson's ratio = 0.25 +# and for a fluid (see the manual of the 2D code, SPECFEM2D, Tables 4.1 and 4.2, and that ratio does not +# depend on whether we are in 2D or in 3D). However in practice a ratio of about 1.5 to 1.7 is often safer +# (for instance for models with a large range of Poisson's ratio values). +# Since the code computes the time step using the Newmark scheme, for LDDRK we will simply +# multiply that time step by this ratio when LDDRK is on and when flag INCREASE_CFL_FOR_LDDRK is true. +INCREASE_CFL_FOR_LDDRK = .true. +RATIO_BY_WHICH_TO_INCREASE_IT = 1.5d0 + +#----------------------------------------------------------- +# +# Visualization +# +#----------------------------------------------------------- + +# save AVS or OpenDX movies +#MOVIE_COARSE saves movie only at corners of elements (SURFACE OR VOLUME) +#MOVIE_COARSE does not work with create_movie_AVS_DX +MOVIE_SURFACE = .false. +MOVIE_VOLUME = .false. +MOVIE_COARSE = .true. +NTSTEP_BETWEEN_FRAMES = 50 +HDUR_MOVIE = 0.d0 + +# save movie in volume. Will save element if center of element is in prescribed volume +# top/bottom: depth in KM, use MOVIE_TOP = -100 to make sure the surface is stored. +# west/east: longitude, degrees East [-180/180] top/bottom: latitute, degrees North [-90/90] +# start/stop: frames will be stored at MOVIE_START + i*NSTEP_BETWEEN_FRAMES, where i=(0,1,2..) and iNSTEP_BETWEEN_FRAMES <= MOVIE_STOP +# movie_volume_type: 1=strain, 2=time integral of strain, 3=\mu*time integral of strain +# type 4 saves the trace and deviatoric stress in the whole volume, 5=displacement, 6=velocity +MOVIE_VOLUME_TYPE = 2 +MOVIE_TOP_KM = -100.0 +MOVIE_BOTTOM_KM = 1000.0 +MOVIE_WEST_DEG = -90.0 +MOVIE_EAST_DEG = 90.0 +MOVIE_NORTH_DEG = 90.0 +MOVIE_SOUTH_DEG = -90.0 +MOVIE_START = 0 +MOVIE_STOP = 40000 + +# save mesh files to check the mesh +SAVE_MESH_FILES = .false. + +# restart files (number of runs can be 1 or higher, choose 1 for no restart files) +NUMBER_OF_RUNS = 1 +NUMBER_OF_THIS_RUN = 1 + +# path to store the local database files on each node +LOCAL_PATH = ./DATABASES_MPI +# temporary wavefield/kernel/movie files +LOCAL_TMP_PATH = ./DATABASES_MPI + +# interval at which we output time step info and max of norm of displacement +NTSTEP_BETWEEN_OUTPUT_INFO = 500 + +#----------------------------------------------------------- +# +# Sources +# +#----------------------------------------------------------- + +# use a (tilted) FORCESOLUTION force point source (or several) instead of a CMTSOLUTION moment-tensor source. +# This can be useful e.g. for asteroid simulations +# in which the source is a vertical force, normal force, tilted force, impact etc. +# If this flag is turned on, the FORCESOLUTION file must be edited by giving: +# - the corresponding time-shift parameter, +# - the half duration parameter of the source, +# - the coordinates of the source, +# - the source time function of the source, +# - the magnitude of the force source, +# - the components of a (non necessarily unitary) direction vector for the force source in the E/N/Z_UP basis. +# The direction vector is made unitary internally in the code and thus only its direction matters here; +# its norm is ignored and the norm of the force used is the factor force source times the source time function. +USE_FORCE_POINT_SOURCE = .true. + +# use monochromatic source time function for CMTSOLUTION moment-tensor source. +# half duration is interpreted as a PERIOD just to avoid changing CMTSOLUTION file format +# default is .false. which uses Heaviside function +USE_MONOCHROMATIC_CMT_SOURCE = .false. + +# uses a sin-squared STF useful for PEGS calculations +# see https://doi.org/10.1016/j.epsl.2020.116150 for definition +USE_SINSQ_STF = .false. + +# print source time function +PRINT_SOURCE_TIME_FUNCTION = .false. + +#----------------------------------------------------------- +# +# Seismograms +# +#----------------------------------------------------------- + +# interval in time steps for temporary writing of seismograms +NTSTEP_BETWEEN_OUTPUT_SEISMOS = 5000000 + +# set to n to reduce the sampling rate of output seismograms by a factor of n +# defaults to 1, which means no down-sampling +NTSTEP_BETWEEN_OUTPUT_SAMPLE = 1 + +# option to save strain seismograms +# this option is useful for strain Green's tensor +# this feature is currently under development +SAVE_SEISMOGRAMS_STRAIN = .false. + +# save seismograms also when running the adjoint runs for an inverse problem +# (usually they are unused and not very meaningful, leave this off in almost all cases) +SAVE_SEISMOGRAMS_IN_ADJOINT_RUN = .false. + +# output format for the seismograms (one can use either or all of the three formats) +OUTPUT_SEISMOS_ASCII_TEXT = .false. +OUTPUT_SEISMOS_SAC_ALPHANUM = .false. +OUTPUT_SEISMOS_SAC_BINARY = .true. +OUTPUT_SEISMOS_ASDF = .false. +OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. + +# rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame +ROTATE_SEISMOGRAMS_RT = .false. + +# decide if main process writes all the seismograms or if all processes do it in parallel +WRITE_SEISMOGRAMS_BY_MAIN = .false. + +# save all seismograms in one large combined file instead of one file per seismogram +# to avoid overloading shared non-local file systems such as LUSTRE or GPFS for instance +SAVE_ALL_SEISMOS_IN_ONE_FILE = .false. +USE_BINARY_FOR_LARGE_FILE = .false. + +# flag to impose receivers at the surface or allow them to be buried +RECEIVERS_CAN_BE_BURIED = .true. + +#----------------------------------------------------------- +# +# Adjoint kernel outputs +# +#----------------------------------------------------------- + +# interval in time steps for reading adjoint traces +# 0 = read the whole adjoint sources at start time +NTSTEP_BETWEEN_READ_ADJSRC = 0 + +# use ASDF format for reading the adjoint sources +READ_ADJSRC_ASDF = .false. + +# this parameter must be set to .true. to compute anisotropic kernels +# in crust and mantle (related to the 21 Cij in geographical coordinates) +# default is .false. to compute isotropic kernels (related to alpha and beta) +ANISOTROPIC_KL = .false. + +# output only transverse isotropic kernels (alpha_v,alpha_h,beta_v,beta_h,eta,rho) +# rather than fully anisotropic kernels when ANISOTROPIC_KL above is set to .true. +# means to save radial anisotropic kernels, i.e., sensitivity kernels for beta_v, beta_h, etc. +SAVE_TRANSVERSE_KL_ONLY = .false. + +# output only the kernels used for the current azimuthally anisotropic inversions of surface waves, +# i.e., bulk_c, bulk_betav, bulk_betah, eta, Gc_prime, Gs_prime and rho +# (Gc' & Gs' which are the normalized Gc & Gs kernels by isotropic \rho\beta of the 1D reference model) +SAVE_AZIMUTHAL_ANISO_KL_ONLY = .false. + +# output approximate Hessian in crust mantle region. +# means to save the preconditioning for gradients, they are cross correlations between forward and adjoint accelerations. +APPROXIMATE_HESS_KL = .false. + +# forces transverse isotropy for all mantle elements +# (default is to use transverse isotropy only between crust and 220) +# means we allow radial anisotropy throughout the whole crust/mantle region +USE_FULL_TISO_MANTLE = .false. + +# output kernel mask to zero out source region +# to remove large values near the sources in the sensitivity kernels +SAVE_SOURCE_MASK = .false. + +# output kernels on a regular grid instead of on the GLL mesh points (a bit expensive) +SAVE_REGULAR_KL = .false. + +# compute steady state kernels for source encoding +STEADY_STATE_KERNEL = .false. +STEADY_STATE_LENGTH_IN_MINUTES = 0.d0 + +#----------------------------------------------------------- + +# Dimitri Komatitsch, July 2014, CNRS Marseille, France: +# added the ability to run several calculations (several earthquakes) +# in an embarrassingly-parallel fashion from within the same run; +# this can be useful when using a very large supercomputer to compute +# many earthquakes in a catalog, in which case it can be better from +# a batch job submission point of view to start fewer and much larger jobs, +# each of them computing several earthquakes in parallel. +# To turn that option on, set parameter NUMBER_OF_SIMULTANEOUS_RUNS to a value greater than 1. +# To implement that, we create NUMBER_OF_SIMULTANEOUS_RUNS MPI sub-communicators, +# each of them being labeled "my_local_mpi_comm_world", and we use them +# in all the routines in "src/shared/parallel.f90", except in MPI_ABORT() because in that case +# we need to kill the entire run. +# When that option is on, of course the number of processor cores used to start +# the code in the batch system must be a multiple of NUMBER_OF_SIMULTANEOUS_RUNS, +# all the individual runs must use the same number of processor cores, +# which as usual is NPROC in the Par_file, +# and thus the total number of processor cores to request from the batch system +# should be NUMBER_OF_SIMULTANEOUS_RUNS * NPROC. +# All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on +# (with exactly four digits). +# +# Imagine you have 10 independent calculations to do, each of them on 100 cores; you have three options: +# +# 1/ submit 10 jobs to the batch system +# +# 2/ submit a single job on 1000 cores to the batch, and in that script create a sub-array of jobs to start 10 jobs, +# each running on 100 cores (see e.g. http://www.schedmd.com/slurmdocs/job_array.html ) +# +# 3/ submit a single job on 1000 cores to the batch, start SPECFEM3D on 1000 cores, create 10 sub-communicators, +# cd into one of 10 subdirectories (called e.g. run0001, run0002,... run0010) depending on the sub-communicator +# your MPI rank belongs to, and run normally on 100 cores using that sub-communicator. +# +# The option below implements 3/. +# +NUMBER_OF_SIMULTANEOUS_RUNS = 1 + +# if we perform simultaneous runs in parallel, if only the source and receivers vary between these runs +# but not the mesh nor the model (velocity and density) then we can also read the mesh and model files +# from a single run in the beginning and broadcast them to all the others; for a large number of simultaneous +# runs for instance when solving inverse problems iteratively this can DRASTICALLY reduce I/Os to disk in the solver +# (by a factor equal to NUMBER_OF_SIMULTANEOUS_RUNS), and reducing I/Os is crucial in the case of huge runs. +# Thus, always set this option to .true. if the mesh and the model are the same for all simultaneous runs. +# In that case there is no need to duplicate the mesh and model file database (the content of the DATABASES_MPI +# directories) in each of the run0001, run0002,... directories, it is sufficient to have one in run0001 +# and the code will broadcast it to the others) +BROADCAST_SAME_MESH_AND_MODEL = .false. + +#----------------------------------------------------------- + +# set to true to use GPUs +GPU_MODE = .true. +# Only used if GPU_MODE = .true. : +GPU_RUNTIME = 1 +# 2 (OpenCL), 1 (Cuda) ou 0 (Compile-time -- does not work if configured with --with-cuda *AND* --with-opencl) +GPU_PLATFORM = NVIDIA +GPU_DEVICE = * + +# set to true to use the ADIOS library for I/Os +ADIOS_ENABLED = .false. +ADIOS_FOR_FORWARD_ARRAYS = .true. +ADIOS_FOR_MPI_ARRAYS = .true. +ADIOS_FOR_ARRAYS_SOLVER = .true. +ADIOS_FOR_SOLVER_MESHFILES = .true. +ADIOS_FOR_AVS_DX = .true. +ADIOS_FOR_KERNELS = .true. +ADIOS_FOR_MODELS = .true. +ADIOS_FOR_UNDO_ATTENUATION = .true. + +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .true. + +# Green function database +GF_DATABASE_ENABLED = .true. +GF_DATABASE_PATH = /scratch/gpfs/TROMP/lsawade/specfem3d_globe/EXAMPLES/green_function_database/global/GFDB/ +GF_SUBSAMPLE_STEP = 0 # 0 = auto (from mesh resolution), or set manually (>= 1) +GF_BUFFER_SIZE = 100 +GF_NEIGHBOR_SHELLS = 1 +DT = 0.1d0 \ No newline at end of file diff --git a/EXAMPLES/green_function_database/global/db_base/DATA/STATIONS b/EXAMPLES/green_function_database/global/db_base/DATA/STATIONS new file mode 100644 index 000000000..aa8dc064a --- /dev/null +++ b/EXAMPLES/green_function_database/global/db_base/DATA/STATIONS @@ -0,0 +1 @@ + SJG IU 18.1091 -66.1500 420.0 0.0 diff --git a/EXAMPLES/green_function_database/global/validation_data/CMTSOLUTION b/EXAMPLES/green_function_database/global/validation_data/CMTSOLUTION new file mode 100644 index 000000000..74c3eed61 --- /dev/null +++ b/EXAMPLES/green_function_database/global/validation_data/CMTSOLUTION @@ -0,0 +1,13 @@ +PDE 1994 6 9 0 33 16.40 -13.8300 -67.5600 637.0 6.9 6.8 NORTHERN BOLIVIA +event name: 060994A +time shift: 29.0000 +half duration: 60.0000 +latitude: -5.8120 +longitude: -75.2700 +depth: 122.6000 +Mrr: -7.590000e+27 +Mtt: 7.750000e+27 +Mpp: -1.600000e+26 +Mrt: -2.503000e+28 +Mrp: 4.200000e+26 +Mtp: -2.480000e+27 diff --git a/EXAMPLES/green_function_database/global/validation_data/FORCESOLUTION b/EXAMPLES/green_function_database/global/validation_data/FORCESOLUTION new file mode 100644 index 000000000..ca391f62e --- /dev/null +++ b/EXAMPLES/green_function_database/global/validation_data/FORCESOLUTION @@ -0,0 +1,11 @@ +FORCE 001 +time shift: 0.0000 +f0: 45.0000 +latitude: -5.8120 +longitude: -75.2700 +depth: 122.6000 +source time function: 0 +factor force source: 1.0d15 +comp dir vect source E: 1.0 +comp dir vect source N: 0.0 +comp dir vect source Z: 0.0 diff --git a/EXAMPLES/green_function_database/global/validation_data/STATIONS b/EXAMPLES/green_function_database/global/validation_data/STATIONS new file mode 100644 index 000000000..c0f5f2ca8 --- /dev/null +++ b/EXAMPLES/green_function_database/global/validation_data/STATIONS @@ -0,0 +1,18 @@ + EFI II -51.6753 -58.0637 110.0 80.0 + HOPE II -54.2836 -36.4879 20.0 0.0 + JTS II 10.2908 -84.9525 340.0 0.0 + NNA II -11.9875 -76.8422 575.0 40.0 + RPN II -27.1267 -109.3344 110.0 0.0 + BOCO IU 4.5869 -74.0432 3137.0 23.0 + DWPF IU 28.1103 -81.4327 -132.0 162.0 + LCO IU -29.0110 -70.7004 2300.0 0.0 + LVC IU -22.6127 -68.9111 2930.0 30.0 + OTAV IU 0.2376 -78.4508 3495.0 15.0 + PAYG IU -0.6742 -90.2861 170.0 100.0 + PTGA IU -0.7308 -59.9666 141.0 96.0 + RCBR IU -5.8274 -35.9014 291.0 109.0 + SAML IU -8.9489 -63.1831 120.0 0.0 + SDV IU 8.8839 -70.6340 1588.0 32.0 + SJG IU 18.1091 -66.1500 420.0 0.0 + TEIG IU 20.2263 -88.2763 15.0 25.0 + TRQA IU -38.0568 -61.9787 439.0 101.0 diff --git a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file index e8e02e434..35c2d0893 100644 --- a/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file +++ b/EXAMPLES/green_function_database/regional/db_base/DATA/Par_file @@ -71,7 +71,7 @@ MODEL = s362ani OCEANS = .true. ELLIPTICITY = .true. TOPOGRAPHY = .true. -GRAVITY = .true. +GRAVITY = .false. ROTATION = .true. ATTENUATION = .true. @@ -444,7 +444,7 @@ HDF5_ENABLED = .true. # Green function database GF_DATABASE_ENABLED = .true. GF_DATABASE_PATH = /scratch/gpfs/TROMP/lsawade/specfem3d_globe/EXAMPLES/green_function_database/regional/GFDB/ -GF_SUBSAMPLE_STEP = 4 # 0 = auto (from mesh resolution), or set manually (>= 1) +GF_SUBSAMPLE_STEP = 0 # 0 = auto (from mesh resolution), or set manually (>= 1) GF_BUFFER_SIZE = 100 GF_NEIGHBOR_SHELLS = 1 DT = 0.1d0 \ No newline at end of file diff --git a/src/shared/parallel.f90 b/src/shared/parallel.f90 index ee51ef6f0..0c242fd81 100644 --- a/src/shared/parallel.f90 +++ b/src/shared/parallel.f90 @@ -1118,8 +1118,39 @@ end subroutine isend_cr !------------------------------------------------------------------------------------------------- ! -! subroutine isend_i(sendbuf, sendcount, dest, sendtag, req) -! end subroutine isend_i + subroutine isend_i(sendbuf, sendcount, dest, sendtag, req) + + use my_mpi + + implicit none + + integer :: sendcount, dest, sendtag, req + integer, dimension(sendcount) :: sendbuf + + integer :: ier + + call MPI_ISEND(sendbuf,sendcount,MPI_INTEGER,dest,sendtag,my_local_mpi_comm_world,req,ier) + + end subroutine isend_i + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine irecv_i(recvbuf, recvcount, dest, recvtag, req) + + use my_mpi + + implicit none + + integer :: recvcount, dest, recvtag, req + integer, dimension(recvcount) :: recvbuf + + integer :: ier + + call MPI_IRECV(recvbuf,recvcount,MPI_INTEGER,dest,recvtag,my_local_mpi_comm_world,req,ier) + + end subroutine irecv_i ! !------------------------------------------------------------------------------------------------- diff --git a/src/shared/read_compute_parameters.f90 b/src/shared/read_compute_parameters.f90 index e64cb1578..2e520b925 100644 --- a/src/shared/read_compute_parameters.f90 +++ b/src/shared/read_compute_parameters.f90 @@ -200,9 +200,9 @@ subroutine rcp_set_compute_parameters() ! Auto-compute GF_SUBSAMPLE_STEP if sentinel value 0 if (GF_DATABASE_ENABLED .and. GF_SUBSAMPLE_STEP == 0) then - ! Nyquist constraint with 0.5x margin (2x oversampling): - ! step = floor(T_min_period / (4 * DT)) - GF_SUBSAMPLE_STEP = int(T_min_period / (4.0d0 * DT)) + ! Nyquist constraint with 0.1x margin (10x oversampling): + ! step = floor(T_min_period / (20 * DT)) + GF_SUBSAMPLE_STEP = int(T_min_period / (20.0d0 * DT)) if (GF_SUBSAMPLE_STEP < 1) GF_SUBSAMPLE_STEP = 1 if (myrank == 0) then print *, 'Green function database:' diff --git a/src/specfem3D/green_function_expand.F90 b/src/specfem3D/green_function_expand.F90 index fe0008249..3f80cb7d5 100644 --- a/src/specfem3D/green_function_expand.F90 +++ b/src/specfem3D/green_function_expand.F90 @@ -205,8 +205,12 @@ subroutine gf_expand_cross_mpi(nspec,nglob,ispec_tagged,ibool, & ! indices to the neighbor rank, which then tags any of its elements that ! contain those nodes. ! -! Uses send_i/recv_i with careful ordering (lower rank sends first) to -! avoid deadlock. +! All interface exchanges are posted with non-blocking isend_i/irecv_i and +! completed with a single wait. This avoids the circular-wait deadlock that +! a per-pair blocking send/recv ordering cannot prevent: each rank iterates +! its interfaces in its own local order, so blocking calls can form a cycle +! (rank A waits on B, B waits on C, C waits on A) even when each individual +! pair is ordered "lower rank sends first". use constants_solver, only: NGLLX,NGLLY,NGLLZ,itag @@ -223,17 +227,17 @@ subroutine gf_expand_cross_mpi(nspec,nglob,ispec_tagged,ibool, & ! local parameters integer :: iinterface, ipoin, iglob, ispec, i, j, k, ier - integer :: nsend, nrecv, neighbor_rank - integer :: ntagged_on_interface + integer :: nsend, neighbor_rank ! node-is-tagged lookup (built from tagged elements) logical, allocatable :: node_tagged(:) - ! send/recv buffers for shared node flags - integer, allocatable :: send_flags(:), recv_flags(:) + ! send/recv buffers for shared node flags, one column per interface so all + ! exchanges can be in flight simultaneously + integer, allocatable :: send_flags(:,:), recv_flags(:,:) + integer, allocatable :: send_req(:), recv_req(:) - ! reverse lookup: node -> element mapping for boundary nodes only - ! We use ibool to find which elements contain received nodes + ! reverse lookup: which boundary nodes were flagged by neighbors logical, allocatable :: node_is_shared(:) ! nothing to do with single process @@ -255,84 +259,74 @@ subroutine gf_expand_cross_mpi(nspec,nglob,ispec_tagged,ibool, & enddo enddo - ! for each MPI interface, exchange flags indicating which shared nodes - ! belong to tagged elements + ! allocate per-interface exchange buffers and request handles + allocate(send_flags(max_nibool_interfaces,num_interfaces), & + recv_flags(max_nibool_interfaces,num_interfaces), & + send_req(num_interfaces), recv_req(num_interfaces), stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating MPI exchange buffers') + + ! pack send buffers: 1 if a shared node belongs to a tagged element, else 0 + send_flags(:,:) = 0 + do iinterface = 1, num_interfaces + do ipoin = 1, nibool_interfaces(iinterface) + iglob = ibool_interfaces(ipoin, iinterface) + if (node_tagged(iglob)) send_flags(ipoin, iinterface) = 1 + enddo + enddo + + ! post all non-blocking receives, then all non-blocking sends + do iinterface = 1, num_interfaces + neighbor_rank = my_neighbors(iinterface) + nsend = nibool_interfaces(iinterface) ! symmetric interface + call irecv_i(recv_flags(1,iinterface), nsend, neighbor_rank, itag, recv_req(iinterface)) + enddo do iinterface = 1, num_interfaces neighbor_rank = my_neighbors(iinterface) nsend = nibool_interfaces(iinterface) - nrecv = nsend ! symmetric interface - - allocate(send_flags(nsend), recv_flags(nrecv), stat=ier) - if (ier /= 0) call exit_MPI(myrank,'Error allocating MPI exchange buffers') + call isend_i(send_flags(1,iinterface), nsend, neighbor_rank, itag, send_req(iinterface)) + enddo - ! pack: 1 if this shared node belongs to a tagged element, 0 otherwise - do ipoin = 1, nsend - iglob = ibool_interfaces(ipoin, iinterface) - if (node_tagged(iglob)) then - send_flags(ipoin) = 1 - else - send_flags(ipoin) = 0 - endif - enddo + ! complete all exchanges + do iinterface = 1, num_interfaces + call wait_req(recv_req(iinterface)) + enddo + do iinterface = 1, num_interfaces + call wait_req(send_req(iinterface)) + enddo - ! exchange using send/recv with ordering to avoid deadlock - if (myrank < neighbor_rank) then - call send_i(send_flags, nsend, neighbor_rank, itag) - call recv_i(recv_flags, nrecv, neighbor_rank, itag) - else - call recv_i(recv_flags, nrecv, neighbor_rank, itag) - call send_i(send_flags, nsend, neighbor_rank, itag) - endif + ! process received flags: mark every boundary node that a neighbor flagged + allocate(node_is_shared(nglob), stat=ier) + if (ier /= 0) call exit_MPI(myrank,'Error allocating node_is_shared') + node_is_shared(:) = .false. - ! process received flags: for shared nodes flagged by the neighbor, - ! find which local elements contain those nodes and tag them - ntagged_on_interface = 0 - do ipoin = 1, nrecv - if (recv_flags(ipoin) == 1) then + do iinterface = 1, num_interfaces + do ipoin = 1, nibool_interfaces(iinterface) + if (recv_flags(ipoin, iinterface) == 1) then iglob = ibool_interfaces(ipoin, iinterface) - ntagged_on_interface = ntagged_on_interface + 1 - ! mark this node so we can find its elements - node_tagged(iglob) = .true. + node_is_shared(iglob) = .true. endif enddo + enddo - ! if any nodes were flagged, find elements containing them - if (ntagged_on_interface > 0) then - ! build a quick set of received flagged node IDs - allocate(node_is_shared(nglob), stat=ier) - if (ier /= 0) call exit_MPI(myrank,'Error allocating node_is_shared') - node_is_shared(:) = .false. - - do ipoin = 1, nrecv - if (recv_flags(ipoin) == 1) then - iglob = ibool_interfaces(ipoin, iinterface) - node_is_shared(iglob) = .true. - endif - enddo - - ! scan all elements: if any corner node is in the received set, tag it - do ispec = 1, nspec - if (ispec_tagged(ispec)) cycle ! already tagged - - ! check 8 corner nodes (sufficient for adjacency — same as xadj/adjncy) - if (node_is_shared(ibool(1,1,1,ispec)) .or. & - node_is_shared(ibool(NGLLX,1,1,ispec)) .or. & - node_is_shared(ibool(NGLLX,NGLLY,1,ispec)) .or. & - node_is_shared(ibool(1,NGLLY,1,ispec)) .or. & - node_is_shared(ibool(1,1,NGLLZ,ispec)) .or. & - node_is_shared(ibool(NGLLX,1,NGLLZ,ispec)) .or. & - node_is_shared(ibool(NGLLX,NGLLY,NGLLZ,ispec)) .or. & - node_is_shared(ibool(1,NGLLY,NGLLZ,ispec))) then - ispec_tagged(ispec) = .true. - endif - enddo - - deallocate(node_is_shared) + ! scan all elements: if any corner node is in the received set, tag it + do ispec = 1, nspec + if (ispec_tagged(ispec)) cycle ! already tagged + + ! check 8 corner nodes (sufficient for adjacency — same as xadj/adjncy) + if (node_is_shared(ibool(1,1,1,ispec)) .or. & + node_is_shared(ibool(NGLLX,1,1,ispec)) .or. & + node_is_shared(ibool(NGLLX,NGLLY,1,ispec)) .or. & + node_is_shared(ibool(1,NGLLY,1,ispec)) .or. & + node_is_shared(ibool(1,1,NGLLZ,ispec)) .or. & + node_is_shared(ibool(NGLLX,1,NGLLZ,ispec)) .or. & + node_is_shared(ibool(NGLLX,NGLLY,NGLLZ,ispec)) .or. & + node_is_shared(ibool(1,NGLLY,NGLLZ,ispec))) then + ispec_tagged(ispec) = .true. endif - - deallocate(send_flags, recv_flags) enddo + deallocate(node_is_shared) + deallocate(send_flags, recv_flags, send_req, recv_req) deallocate(node_tagged) end subroutine gf_expand_cross_mpi diff --git a/src/specfem3D/green_function_metadata.F90 b/src/specfem3D/green_function_metadata.F90 index aa57a4ea3..2580e976e 100644 --- a/src/specfem3D/green_function_metadata.F90 +++ b/src/specfem3D/green_function_metadata.F90 @@ -213,7 +213,7 @@ subroutine gf_write_mesh_info() use constants, only: CUSTOM_REAL,NGLLX,IMAIN,MAX_STRING_LEN,NR_DENSITY - use specfem_par, only: myrank, NSTEP, DT, & + use specfem_par, only: myrank, NSTEP, DT, t0, & ibathy_topo, NX_BATHY_VAL, NY_BATHY_VAL, & nspl_ellip, rspl_ellip, ellipicity_spline, ellipicity_spline2, & scale_displ @@ -370,6 +370,19 @@ subroutine gf_write_mesh_info() call h5aclose_f(attr_id, hdferr) call h5sclose_f(aspace_id, hdferr) + ! t0: time of the first time step relative to the source origin. + ! The simulation time axis is timeval(it) = (it-1)*DT - t0, so the first + ! stored (subsampled) sample sits at -t0 + (first_snap-1)*DT. The GF + ! reconstruction needs t0 to place its trace on an absolute time axis that + ! is consistent with a forward simulation; storing it here makes the + ! database self-contained (no need to parse output_solver.txt). + call h5screate_simple_f(1, adim, aspace_id, hdferr) + call h5acreate_f(fid, 't0', H5T_NATIVE_DOUBLE, aspace_id, attr_id, hdferr) + attr_dp(1) = t0 + call h5awrite_f(attr_id, H5T_NATIVE_DOUBLE, attr_dp, adim, hdferr) + call h5aclose_f(attr_id, hdferr) + call h5sclose_f(aspace_id, hdferr) + call h5screate_simple_f(1, adim, aspace_id, hdferr) call h5acreate_f(fid, 'nstep', H5T_NATIVE_INTEGER, aspace_id, attr_id, hdferr) attr_int(1) = NSTEP diff --git a/utils/green_function/gf_cross_validate.py b/utils/green_function/gf_cross_validate.py index e1b9a20ee..0e83dbe67 100644 --- a/utils/green_function/gf_cross_validate.py +++ b/utils/green_function/gf_cross_validate.py @@ -33,6 +33,7 @@ import numpy as np from scipy.signal import fftconvolve +from scipy.integrate import cumulative_trapezoid try: import h5py @@ -539,7 +540,9 @@ def compute_strain(displ, xi, eta, gamma, Jinv): def read_mesh_info(gf_db): """Read mesh timing parameters from the GF database. - Returns dict with keys: dt, nstep, subsample_step, nt_sub. + Returns dict with keys: dt, nstep, subsample_step, nt_sub, and t0 (the + time of the first time step relative to the source origin; None for older + databases that did not store it). """ with h5py.File(gf_db / "mesh_info.h5", "r") as f: info = { @@ -547,6 +550,8 @@ def read_mesh_info(gf_db): "nstep": int(np.asarray(f.attrs["nstep"]).flat[0]), "subsample_step": int(np.asarray(f.attrs["subsample_step"]).flat[0]), "nt_sub": int(np.asarray(f.attrs["nt_subsampled"]).flat[0]), + "t0": (float(np.asarray(f.attrs["t0"]).flat[0]) + if "t0" in f.attrs else None), } return info @@ -628,9 +633,14 @@ def reconstruct_cmt(displ, xi, eta, gamma, gf_db, morton_hex, for i in range(3): gf[i] = np.sum(weights[:, None] * strain[:, :, i].T, axis=0) - # Time-integrate: Gaussian -> Heaviside STF conversion + # Time-integrate: Gaussian -> Heaviside STF conversion. + # Trapezoidal rule (second-order, phase-correct). A left-endpoint Riemann + # sum (np.cumsum * dt) is first-order and lags the true integral by half a + # sample (dt_sub/2); on the coarse subsampled grid that is ~0.2 s, a + # visible sub-sample phase error. The integrand is band-limited well below + # the subsampled Nyquist, so the trapezoid is essentially exact here. for i in range(3): - gf[i] = np.cumsum(gf[i]) * dt_sub + gf[i] = cumulative_trapezoid(gf[i], dx=dt_sub, initial=0.0) return gf, cmt @@ -863,7 +873,7 @@ def plot_comparison(time_common, gf_traces, fwd_traces, comps, comp_labels, residuals, station, source_type, morton_hex, xi, eta, gamma, highpass_period, lowpass_period, output_plot): """Plot GF vs forward comparison and save.""" - fig, axes = plt.subplots(3, 1, figsize=(14, 10), sharex=True) + fig, axes = plt.subplots(3, 1, figsize=(10.5, 7.5), sharex=True) for ax, comp in zip(axes, comps): ax.plot(time_common, fwd_traces[comp], "b-", linewidth=0.8, @@ -1017,24 +1027,13 @@ def cross_validate( # 7. Read forward seismograms fwd_traces, channel_prefix = read_forward_seismograms(fwd_dir, station) - # 8. Construct GF time axis - t0_gf = None - gf_solver_candidates = [ - gf_solver_output, - fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver.txt", - fwd_dir.parent / "simulations" / station / "OUTPUT_FILES" / "output_solver_N.txt", - ] - for gf_solver_path in gf_solver_candidates: - if gf_solver_path is not None and Path(gf_solver_path).exists(): - gf_solver_text = Path(gf_solver_path).read_text() - m = re.search(r"start time\s*:\s*([-\dE.+]+)", gf_solver_text) - if m: - t0_gf = -float(m.group(1)) - break - if t0_gf is None: - t0_gf = sta_meta["hdur"] * 5.0 - print(f" WARNING: could not find GF t0, estimating t0_gf={t0_gf:.4f}") - + # 8. Construct GF time axis. + # t0_gf is the time of the first time step relative to the source origin + # (simulation time is (it-1)*dt - t0). It sets the absolute time origin of + # the reconstructed trace, so an error here is a rigid time shift between + # the GF and forward traces. + t0_gf = mesh.get("t0") + time_gf = np.array([(isnap * subsample_step - 1) * dt - t0_gf for isnap in range(1, nt_sub + 1)]) diff --git a/utils/green_function/xvalidate.py b/utils/green_function/xvalidate.py index 0101ff09b..b5164a245 100644 --- a/utils/green_function/xvalidate.py +++ b/utils/green_function/xvalidate.py @@ -134,7 +134,7 @@ def main(): station=station, force=True, lowpass_period=lowpass_period, - highpass_period=0.004, + # highpass_period=0.004, output=output_dir / "xvalidate_force.svg", ) @@ -150,7 +150,7 @@ def main(): force=False, cmtsolution=cmtsolution, lowpass_period=lowpass_period, - highpass_period=0.004, + # highpass_period=0.004, output=output_dir / "xvalidate_cmt.svg", ) From 13044c2af993c504735f0b80b15ec551e6791e5e Mon Sep 17 00:00:00 2001 From: Lucas Sawade Date: Thu, 25 Jun 2026 18:47:53 -0400 Subject: [PATCH 20/20] Updated the .gitignore too ignore __pycache__ --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 74309c44d..09ae9a991 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ slurm* submit* # Mac directory file -.DS_Store \ No newline at end of file +.DS_Store +__pycache__ \ No newline at end of file