Kokkos support for Lagrange and Monomial elements

Makefile.am

@@ -44,6 +44,111 @@ AM_CPPFLAGS      = -DLIBMESH_IS_COMPILING_ITSELF \
                    $(libmesh_optional_INCLUDES) \
                    -I$(top_builddir)/include # required for libmesh_version.h
 
+if LIBMESH_ENABLE_KOKKOS
+KOKKOS_LIBMESH_DBG_OBJECTS = src/base/dof_map_kokkos_dbg.lo src/mesh/mesh_base_kokkos_dbg.lo
+KOKKOS_LIBMESH_DEVEL_OBJECTS = src/base/dof_map_kokkos_devel.lo src/mesh/mesh_base_kokkos_devel.lo
+KOKKOS_LIBMESH_OPT_OBJECTS = src/base/dof_map_kokkos_opt.lo src/mesh/mesh_base_kokkos_opt.lo
+KOKKOS_LIBMESH_PROF_OBJECTS = src/base/dof_map_kokkos_prof.lo src/mesh/mesh_base_kokkos_prof.lo
+KOKKOS_LIBMESH_OPROF_OBJECTS = src/base/dof_map_kokkos_oprof.lo src/mesh/mesh_base_kokkos_oprof.lo
+KOKKOS_CALCULATOR_DBG_OBJECTS = src/apps/L2system_kokkos_dbg.o
+KOKKOS_CALCULATOR_DEVEL_OBJECTS = src/apps/L2system_kokkos_devel.o
+KOKKOS_CALCULATOR_OPT_OBJECTS = src/apps/L2system_kokkos_opt.o
+
+include/libmesh/dof_map_kokkos.h:
+	@$(MAKE) -C $(top_builddir)/include/libmesh dof_map_kokkos.h
+
+include/libmesh/mesh_base_kokkos.h:
+	@$(MAKE) -C $(top_builddir)/include/libmesh mesh_base_kokkos.h
+
+src/base/dof_map_kokkos_dbg.lo: src/base/dof_map_kokkos.K include/libmesh/dof_map_kokkos.h include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DBG) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DBG) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/base/dof_map_kokkos_devel.lo: src/base/dof_map_kokkos.K include/libmesh/dof_map_kokkos.h include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DEVEL) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DEVEL) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/base/dof_map_kokkos_opt.lo: src/base/dof_map_kokkos.K include/libmesh/dof_map_kokkos.h include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_OPT) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_OPT) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/base/dof_map_kokkos_prof.lo: src/base/dof_map_kokkos.K include/libmesh/dof_map_kokkos.h include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_PROF) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_PROF) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/base/dof_map_kokkos_oprof.lo: src/base/dof_map_kokkos.K include/libmesh/dof_map_kokkos.h include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_OPROF) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_OPROF) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/mesh/mesh_base_kokkos_dbg.lo: src/mesh/mesh_base_kokkos.K include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DBG) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DBG) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/mesh/mesh_base_kokkos_devel.lo: src/mesh/mesh_base_kokkos.K include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DEVEL) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DEVEL) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/mesh/mesh_base_kokkos_opt.lo: src/mesh/mesh_base_kokkos.K include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_OPT) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_OPT) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/mesh/mesh_base_kokkos_prof.lo: src/mesh/mesh_base_kokkos.K include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_PROF) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_PROF) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/mesh/mesh_base_kokkos_oprof.lo: src/mesh/mesh_base_kokkos.K include/libmesh/mesh_base_kokkos.h
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX --mode=compile $(KOKKOS_CXX) \
+	  $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_OPROF) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_OPROF) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/L2system_kokkos_dbg.o: src/apps/L2system_kokkos.K
+	$(KOKKOS_CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DBG) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DBG) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/L2system_kokkos_devel.o: src/apps/L2system_kokkos.K
+	$(KOKKOS_CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_DEVEL) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_DEVEL) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/L2system_kokkos_opt.o: src/apps/L2system_kokkos.K
+	$(KOKKOS_CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(CPPFLAGS_OPT) $(AM_CPPFLAGS) $(CPPFLAGS) $(KOKKOS_MPI_CPPFLAGS) $(KOKKOS_CPPFLAGS) \
+	  $(CXXFLAGS_OPT) $(AM_CXXFLAGS) $(CXXFLAGS) $(KOKKOS_CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+endif
+
 LIBS             = $(libmesh_optional_LIBS) $(libmesh_precision_LIBS)
 
 # additional files which must be included in 'make dist'
@@ -268,7 +373,7 @@ if LIBMESH_DBG_MODE
   libmesh_dbg_la_CPPFLAGS = $(CPPFLAGS_DBG) $(AM_CPPFLAGS)
   libmesh_dbg_la_CXXFLAGS = $(CXXFLAGS_DBG)
   libmesh_dbg_la_CFLAGS   = $(CFLAGS_DBG)
-  libmesh_dbg_la_LIBADD   = contrib/libcontrib_dbg.la $(LIBS)
+  libmesh_dbg_la_LIBADD   = contrib/libcontrib_dbg.la $(KOKKOS_LIBMESH_DBG_OBJECTS) $(LIBS)
 endif
 
 if LIBMESH_DEVEL_MODE
@@ -277,7 +382,7 @@ if LIBMESH_DEVEL_MODE
   libmesh_devel_la_CPPFLAGS = $(CPPFLAGS_DEVEL) $(AM_CPPFLAGS)
   libmesh_devel_la_CXXFLAGS = $(CXXFLAGS_DEVEL)
   libmesh_devel_la_CFLAGS   = $(CFLAGS_DEVEL)
-  libmesh_devel_la_LIBADD   = contrib/libcontrib_devel.la $(LIBS)
+  libmesh_devel_la_LIBADD   = contrib/libcontrib_devel.la $(KOKKOS_LIBMESH_DEVEL_OBJECTS) $(LIBS)
 endif
 
 if LIBMESH_OPT_MODE
@@ -286,7 +391,7 @@ if LIBMESH_OPT_MODE
   libmesh_opt_la_CPPFLAGS = $(CPPFLAGS_OPT) $(AM_CPPFLAGS)
   libmesh_opt_la_CXXFLAGS = $(CXXFLAGS_OPT)
   libmesh_opt_la_CFLAGS   = $(CFLAGS_OPT)
-  libmesh_opt_la_LIBADD   = contrib/libcontrib_opt.la $(LIBS)
+  libmesh_opt_la_LIBADD   = contrib/libcontrib_opt.la $(KOKKOS_LIBMESH_OPT_OBJECTS) $(LIBS)
 endif
 
 if LIBMESH_PROF_MODE
@@ -295,7 +400,7 @@ if LIBMESH_PROF_MODE
   libmesh_prof_la_CPPFLAGS = $(CPPFLAGS_PROF) $(AM_CPPFLAGS)
   libmesh_prof_la_CXXFLAGS = $(CXXFLAGS_PROF)
   libmesh_prof_la_CFLAGS   = $(CFLAGS_PROF)
-  libmesh_prof_la_LIBADD   = contrib/libcontrib_prof.la $(LIBS)
+  libmesh_prof_la_LIBADD   = contrib/libcontrib_prof.la $(KOKKOS_LIBMESH_PROF_OBJECTS) $(LIBS)
 endif
 
 if LIBMESH_OPROF_MODE
@@ -304,7 +409,7 @@ if LIBMESH_OPROF_MODE
   libmesh_oprof_la_CPPFLAGS = $(CPPFLAGS_OPROF) $(AM_CPPFLAGS)
   libmesh_oprof_la_CXXFLAGS = $(CXXFLAGS_OPROF)
   libmesh_oprof_la_CFLAGS   = $(CFLAGS_OPROF)
-  libmesh_oprof_la_LIBADD   = contrib/libcontrib_oprof.la $(LIBS)
+  libmesh_oprof_la_LIBADD   = contrib/libcontrib_oprof.la $(KOKKOS_LIBMESH_OPROF_OBJECTS) $(LIBS)
 endif
 
 # Make sure we build the library before we test it
@@ -447,6 +552,85 @@ meshtool_dbg_CXXFLAGS   = $(CXXFLAGS_DBG)
 meshtool_dbg_LDADD      = libmesh_dbg.la
 
 # calculator
+if LIBMESH_ENABLE_KOKKOS
+opt_programs                 += calculator-kokkos-opt
+calculator_kokkos_opt_SOURCES = src/apps/calculator_kokkos.K
+calculator_kokkos_opt_SOURCES += src/apps/L2system.C src/apps/L2system.h
+calculator_kokkos_opt_CPPFLAGS = $(CPPFLAGS_OPT) $(AM_CPPFLAGS)
+calculator_kokkos_opt_CXXFLAGS = $(CXXFLAGS_OPT)
+calculator_kokkos_opt_LDFLAGS  = $(AM_LDFLAGS) $(KOKKOS_LDFLAGS)
+calculator_kokkos_opt_LDADD    = libmesh_opt.la \
+                                 $(KOKKOS_LIBMESH_OPT_OBJECTS) \
+                                 $(KOKKOS_CALCULATOR_OPT_OBJECTS) \
+                                 $(KOKKOS_LIBS)
+calculator_kokkos_opt_LINK = \
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(KOKKOS_CXX) \
+	$(calculator_kokkos_opt_LDFLAGS) $(LDFLAGS) -o $@
+
+src/apps/calculator_kokkos_opt-calculator_kokkos.o: src/apps/calculator_kokkos.K
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_opt_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_opt_CXXFLAGS) $(CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/calculator_kokkos_opt-L2system.o: src/apps/L2system.C
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_opt_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_opt_CXXFLAGS) $(CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+devel_programs                   += calculator-kokkos-devel
+calculator_kokkos_devel_SOURCES   = src/apps/calculator_kokkos.K
+calculator_kokkos_devel_SOURCES  += src/apps/L2system.C src/apps/L2system.h
+calculator_kokkos_devel_CPPFLAGS  = $(CPPFLAGS_DEVEL) $(AM_CPPFLAGS)
+calculator_kokkos_devel_CXXFLAGS  = $(CXXFLAGS_DEVEL)
+calculator_kokkos_devel_LDFLAGS   = $(AM_LDFLAGS) $(KOKKOS_LDFLAGS)
+calculator_kokkos_devel_LDADD     = libmesh_devel.la \
+                                    $(KOKKOS_LIBMESH_DEVEL_OBJECTS) \
+                                    $(KOKKOS_CALCULATOR_DEVEL_OBJECTS) \
+                                    $(KOKKOS_LIBS)
+calculator_kokkos_devel_LINK = \
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(KOKKOS_CXX) \
+	$(calculator_kokkos_devel_LDFLAGS) $(LDFLAGS) -o $@
+
+src/apps/calculator_kokkos_devel-calculator_kokkos.o: src/apps/calculator_kokkos.K
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_devel_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_devel_CXXFLAGS) $(CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/calculator_kokkos_devel-L2system.o: src/apps/L2system.C
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_devel_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_devel_CXXFLAGS) $(CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+dbg_programs                 += calculator-kokkos-dbg
+calculator_kokkos_dbg_SOURCES = src/apps/calculator_kokkos.K
+calculator_kokkos_dbg_SOURCES += src/apps/L2system.C src/apps/L2system.h
+calculator_kokkos_dbg_CPPFLAGS = $(CPPFLAGS_DBG) $(AM_CPPFLAGS)
+calculator_kokkos_dbg_CXXFLAGS = $(CXXFLAGS_DBG)
+calculator_kokkos_dbg_LDFLAGS  = $(AM_LDFLAGS) $(KOKKOS_LDFLAGS)
+calculator_kokkos_dbg_LDADD    = libmesh_dbg.la \
+                                 $(KOKKOS_LIBMESH_DBG_OBJECTS) \
+                                 $(KOKKOS_CALCULATOR_DBG_OBJECTS) \
+                                 $(KOKKOS_LIBS)
+calculator_kokkos_dbg_LINK = \
+	$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(KOKKOS_CXX) \
+	$(calculator_kokkos_dbg_LDFLAGS) $(LDFLAGS) -o $@
+
+src/apps/calculator_kokkos_dbg-calculator_kokkos.o: src/apps/calculator_kokkos.K
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_dbg_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_dbg_CXXFLAGS) $(CXXFLAGS) \
+	  -x c++ -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+
+src/apps/calculator_kokkos_dbg-L2system.o: src/apps/L2system.C
+	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	  $(calculator_kokkos_dbg_CPPFLAGS) $(CPPFLAGS) \
+	  $(calculator_kokkos_dbg_CXXFLAGS) $(CXXFLAGS) \
+	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<
+else
 opt_programs          += calculator-opt
 calculator_opt_SOURCES    = src/apps/calculator.C
 calculator_opt_SOURCES   += src/apps/L2system.C src/apps/L2system.h
@@ -467,6 +651,7 @@ calculator_dbg_SOURCES   += src/apps/L2system.C src/apps/L2system.h
 calculator_dbg_CPPFLAGS   = $(CPPFLAGS_DBG) $(AM_CPPFLAGS)
 calculator_dbg_CXXFLAGS   = $(CXXFLAGS_DBG)
 calculator_dbg_LDADD      = libmesh_dbg.la
+endif
 
 # compare
 opt_programs          += compare-opt