Merge pull request #97 from kokkos/elisabethgiem/change-tmr-cmake-option

Change TMR/DMR cmake option

Author: nmm0

CMakeLists.txt

@@ -88,9 +88,16 @@ kr_option(KR_ENABLE_HDF5_PARALLEL "use parallel version of HDF5" OFF KR_ENABLE_H
 #Exec space options
 kr_option(KR_ENABLE_EXEC_SPACES "enable resilient execution spaces" OFF)
 kr_option(KR_ENABLE_OPENMP_EXEC_SPACE "enable the resilient OpenMP execution space" ON "KR_ENABLE_EXEC_SPACES;KR_OPENMP_DEVICE_ENABLED")
-kr_option(KR_ENABLE_DMR "enable double modular redundancy" OFF "KR_ENABLE_OPENMP_EXEC_SPACE")
-kr_option(KR_ENABLE_TMR "enable triple modular redundancy" ON "KR_ENABLE_OPENMP_EXEC_SPACE;NOT KR_ENABLE_DMR")
-kr_option(KR_ENABLE_WRAPPER "enable kernel fusing" OFF "KR_ENABLE_OPENMP_EXEC_SPACE")
+if(KR_ENABLE_OPENMP_EXEC_SPACE)
+  set(KR_MODULAR_REDUNDANCY TRIPLE CACHE STRING "choose level of modular redundancy")
+  set_property(CACHE KR_MODULAR_REDUNDANCY PROPERTY STRINGS OFF DOUBLE TRIPLE)
+  if (KR_MODULAR_REDUNDANCY STREQUAL TRIPLE)
+    target_compile_definitions(resilience PUBLIC KR_TRIPLE_MODULAR_REDUNDANCY)
+  elseif (KR_MODULAR_REDUNDANCY STREQUAL DOUBLE)
+    target_compile_definitions(resilience PUBLIC KR_DOUBLE_MODULAR_REDUNDANCY)
+  endif()
+endif()
+kr_option(KR_KERNEL_FUSING "enable kernel fusing" OFF "KR_ENABLE_OPENMP_EXEC_SPACE;KR_MODULAR_REDUNDANCY")
 
 # VeloC backend
 if (KR_ENABLE_VELOC_BACKEND)

src/resilience/exec_space/openMP/Resilient_OpenMP_Parallel_For.hpp

@@ -120,7 +120,7 @@ class ParallelFor< FunctorType
 
   using surrogate_policy = Kokkos::RangePolicy < Kokkos::OpenMP, WorkTag, LaunchBounds>;
 
-#ifdef KR_ENABLE_WRAPPER
+#ifdef KR_KERNEL_FUSING
   auto make_wrapper (int64_t work_size, int64_t offset, const FunctorType &functor_copy_0, const FunctorType &functor_copy_1) const{
     if constexpr (std::is_void_v<WorkTag>){
       auto wrapper_functor = [&, work_size, offset](int64_t i){
@@ -173,7 +173,8 @@ class ParallelFor< FunctorType
     bool success = 0; //! This bool indicates that all views successfully reached a consensus.
 
       surrogate_policy wrapper_policy;
-#ifdef KR_ENABLE_TMR
+      
+#if KR_TRIPLE_MODULAR_REDUNDANCY
       wrapper_policy = surrogate_policy(m_policy.begin(), m_policy.end());
       // Trigger Subscriber constructors
       KokkosResilience::ResilientDuplicatesSubscriber::in_resilient_parallel_loop = true;
@@ -182,7 +183,7 @@ class ParallelFor< FunctorType
       KokkosResilience::ResilientDuplicatesSubscriber::in_resilient_parallel_loop = false;
 #endif
 
-#ifdef KR_ENABLE_WRAPPER
+#ifdef KR_KERNEL_FUSING
       auto work_size = m_policy.end() - m_policy.begin();
       auto offset = m_policy.begin();
       wrapper_policy = surrogate_policy(0, 3 * work_size );
@@ -197,10 +198,8 @@ class ParallelFor< FunctorType
 
 #endif
 
-#ifdef KR_ENABLE_TMR
-
+#ifdef KR_TRIPLE_MODULAR_REDUNDANCY 
       // TMR execution with no wrapper scheduling
-
       Impl::ParallelFor< decltype(m_functor) , surrogate_policy, Kokkos::OpenMP > closure0( m_functor , wrapper_policy );
       Impl::ParallelFor< decltype(m_functor) , surrogate_policy, Kokkos::OpenMP > closure1( functor_copy_0 , wrapper_policy );
       Impl::ParallelFor< decltype(m_functor) , surrogate_policy, Kokkos::OpenMP > closure2( functor_copy_1 , wrapper_policy );
@@ -223,8 +222,7 @@ class ParallelFor< FunctorType
 
 #endif
 
-#ifdef KR_ENABLE_DMR
-
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
       //DMR with failover to TMR on error
       wrapper_policy = surrogate_policy(m_policy.begin(), m_policy.end());
 
@@ -270,7 +268,8 @@ class ParallelFor< FunctorType
       }
       KokkosResilience::clear_duplicates_map();
 #endif
-#ifdef KR_ENABLE_WRAPPER
+
+#ifdef KR_KERNEL_FUSION
 
       // TMR with kernel fusion
       // Functor is fused, with iterations bound to duplicated functors in 3x range

src/resilience/exec_space/openMP/Resilient_OpenMP_Subscriber.cpp

@@ -49,7 +49,7 @@
 namespace KokkosResilience {
 
 bool ResilientDuplicatesSubscriber::in_resilient_parallel_loop = false;
-#ifdef KR_ENABLE_DMR
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
 bool ResilientDuplicatesSubscriber::dmr_failover_to_tmr = false;
 #endif
 }

src/resilience/exec_space/openMP/Resilient_OpenMP_Subscriber.hpp

@@ -208,7 +208,7 @@ struct CombineDuplicates: public CombineDuplicatesBase
   bool execute() override
   { 
     success() = 1;
-#ifdef KR_ENABLE_DMR
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
     if (duplicate_count < 1){
       Kokkos::abort("Aborted in CombineDuplicates, no duplicate created");
     }
@@ -238,7 +238,7 @@ struct CombineDuplicates: public CombineDuplicatesBase
   KOKKOS_INLINE_FUNCTION
   void operator ()(Args&&... its) const{ //function parameter pack
 
-#ifdef KR_ENABLE_DMR
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
     //Indicates dmr_failover_to_tmr tripped
     if(duplicate_count == 2 ){
       //Main combiner begin, dmr failover has tripped into TMR
@@ -280,13 +280,12 @@ struct CombineDuplicates: public CombineDuplicatesBase
 
   void inject_error() override
   {
-#ifdef KR_ENABLE_TMR
+#ifdef KR_TRIPLE_MODULAR_REDUNDANCY
     //Any-dimensional TMR error injector
     size_t total_extent = 1;
     for(int i=0; i<= (int)rank; i++){
       total_extent = total_extent * original.extent(i);	    
     }
-
     //requires error in range, unless view size too small
     if (total_extent !=1 && (ErrorInject::global_next_inject > total_extent))
     {
@@ -337,7 +336,7 @@ struct ResilientDuplicatesSubscriber {
   // Gating for using subscriber only inside resilient parallel loops
   static bool in_resilient_parallel_loop;
 
-#ifdef KR_ENABLE_DMR
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
   static bool dmr_failover_to_tmr;
 
 #endif
@@ -382,7 +381,7 @@ struct ResilientDuplicatesSubscriber {
     if (inserted || extents_resized) {
       res.original = original;
 
-#ifdef KR_ENABLE_DMR
+#ifdef KR_DOUBLE_MODULAR_REDUNDANCY
       if (dmr_failover_to_tmr){
         // Create second copy
         set_duplicate_view(res.copy[1], original, 1);

tests/TestOpenMPResilientExecution.cpp

@@ -102,11 +102,9 @@ TEST(TestResOpenMP, TestResilientForDouble)
 {
 
   KokkosResilience::global_error_settings = KokkosResilience::Error(0.001);
-  
   // Allocate y, x vectors.
   ResilientView<double*> y( "y", N );
   ResilientView<double*> x( "x", N );
-
   //Integer vector 1 long to count data accesses, because scalar view bugs (previously)
   ResilientView<int*> counter( "DataAccesses", 1);
 
@@ -117,11 +115,10 @@ TEST(TestResOpenMP, TestResilientForDouble)
     y ( i ) = i;
     Kokkos::atomic_inc(&counter(0));
   });
-
   //reset global error settings
+  KokkosResilience::print_total_error_time();
   KokkosResilience::ErrorInject::error_counter=0;
   KokkosResilience::global_error_settings.reset();
-  KokkosResilience::print_total_error_time();
   KokkosResilience::clear_duplicates_cache();
 
   Kokkos::deep_copy(x, y);