Add floating point atomics

vtkm/Atomic.h

@@ -225,6 +225,44 @@ VTKM_EXEC_CONT inline bool AtomicCompareExchangeImpl(T* addr,
     return false;
   }
 }
+#if __CUDA_ARCH__ < 200
+VTKM_EXEC_CONT inline vtkm::Float32 vtkmAtomicAddImpl(vtkm::Float32* address,
+                                                      vtkm::Float32 value,
+                                                      vtkm::MemoryOrder order)
+{
+  AtomicStoreFence(order);
+  vtkm::UInt32 assumed;
+  vtkm::UInt32 old = __float_as_int(*address);
+  do
+  {
+    assumed = old;
+    old = atomicCAS(reinterpret_cast<volatile vtkm::UInt32*>(address),
+                    assumed,
+                    __float_as_int(__int_as_float(assumed) + value));
+  } while (assumed != old);
+  AtomicLoadFence(order);
+  return __int_as_float(old);
+}
+#endif
+#if __CUDA_ARCH__ < 600
+VTKM_EXEC_CONT inline vtkm::Float64 vtkmAtomicAdd(vtkm::Float64* address,
+                                                  vtkm::Float64 value,
+                                                  vtkm::MemoryOrder order)
+{
+  AtomicStoreFence(order);
+  vtkm::UInt64 assumed;
+  vtkm::UInt64 old = __double_as_longlong(*address);
+  do
+  {
+    assumed = old;
+    old = atomicCAS(reinterpret_cast<volatile vtkm::UInt64*>(address),
+                    assumed,
+                    __double_as_longlong(__longlong_as_double(assumed) + value));
+  } while (assumed != old);
+  AtomicLoadFence(order);
+  return __longlong_as_double(old);
+}
+#endif
 }
 } // namespace vtkm::detail
 
@@ -532,6 +570,39 @@ VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt32, LONG, )
 VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt64, LONG64, 64)
 
 #undef VTKM_ATOMIC_OPS_FOR_TYPE
+
+VTKM_EXEC_CONT inline vtkm::Float32 AtomicAddImpl(vtkm::Float32* address,
+                                                  vtkm::Float32 value,
+                                                  vtkm::MemoryOrder vtkmNotUsed(order))
+{
+  LONG assumed;
+  LONG old = BitCast<LONG>(*address);
+  do
+  {
+    assumed = old;
+    old = _InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(address),
+                                      BitCast<LONG>(BitCast<vtkm::Float32>(assumed) + value),
+                                      assumed);
+  } while (assumed != old);
+  return BitCast<vtkm::Float32>(old);
+}
+
+VTKM_EXEC_CONT inline vtkm::Float64 AtomicAddImpl(vtkm::Float64* address,
+                                                  vtkm::Float64 value,
+                                                  vtkm::MemoryOrder vtkmNotUsed(order))
+{
+  LONG64 assumed;
+  LONG64 old = BitCast<LONG64>(*address);
+  do
+  {
+    assumed = old;
+    old = _InterlockedCompareExchange64(reinterpret_cast<volatile LONG64*>(address),
+                                        BitCast<LONG64>(BitCast<vtkm::Float64>(assumed) + value),
+                                        assumed);
+  } while (assumed != old);
+  return BitCast<vtkm::Float64>(old);
+}
+
 }
 } // namespace vtkm::detail
 
@@ -585,6 +656,52 @@ VTKM_EXEC_CONT inline T AtomicAddImpl(T* addr, T arg, vtkm::MemoryOrder order)
   return __atomic_fetch_add(addr, arg, GccAtomicMemOrder(order));
 }
 
+#include <vtkmstd/bit_cast.h>
+
+// TODO: Use enable_if to write one version for both Float32 and Float64.
+VTKM_EXEC_CONT inline vtkm::Float32 AtomicAddImpl(vtkm::Float32* addr,
+                                                  vtkm::Float32 arg,
+                                                  vtkm::MemoryOrder order)
+{
+  vtkm::UInt32 expected = vtkmstd::bit_cast<vtkm::UInt32>(*addr);
+  vtkm::UInt32 desired;
+
+  do
+  {
+    desired = vtkmstd::bit_cast<vtkm::UInt32>(vtkmstd::bit_cast<vtkm::Float32>(expected) + arg);
+  } while (
+    !__atomic_compare_exchange_n(reinterpret_cast<vtkm::UInt32*>(addr),
+                                 &expected, // reloads expected with *addr prior to the operation
+                                 desired,
+                                 false,
+                                 GccAtomicMemOrder(order),
+                                 GccAtomicMemOrder(order)));
+  // return the "old" value that was in the memory.
+  return vtkmstd::bit_cast<vtkm::Float32>(expected);
+}
+
+// TODO: Use enable_if to write one version for both Float32 and Float64.
+VTKM_EXEC_CONT inline vtkm::Float64 AtomicAddImpl(vtkm::Float64* addr,
+                                                  vtkm::Float64 arg,
+                                                  vtkm::MemoryOrder order)
+{
+  vtkm::UInt64 expected = vtkmstd::bit_cast<vtkm::UInt64>(*addr);
+  vtkm::UInt64 desired;
+
+  do
+  {
+    desired = vtkmstd::bit_cast<vtkm::UInt64>(vtkmstd::bit_cast<vtkm::Float64>(expected) + arg);
+  } while (
+    !__atomic_compare_exchange_n(reinterpret_cast<vtkm::UInt64*>(addr),
+                                 &expected, // reloads expected with *addr prior to the operation
+                                 desired,
+                                 false,
+                                 GccAtomicMemOrder(order),
+                                 GccAtomicMemOrder(order)));
+  // return the "old" value that was in the memory.
+  return vtkmstd::bit_cast<vtkm::Float64>(expected);
+}
+
 template <typename T>
 VTKM_EXEC_CONT inline T AtomicAndImpl(T* addr, T mask, vtkm::MemoryOrder order)
 {
@@ -706,7 +823,7 @@ VTKM_EXEC_CONT inline T AtomicAdd(
 {
   return detail::AtomicAddImpl(pointer, operand, order);
 }
-template <typename T>
+template <typename T, typename std::enable_if<std::is_integral<T>::value>::type* = nullptr>
 VTKM_EXEC_CONT inline T AtomicAdd(
   T* pointer,
   detail::OppositeSign<T> operand,
@@ -846,7 +963,7 @@ VTKM_EXEC_CONT inline T AtomicNot(
 /// pointing to an object on the stack).
 ///
 template <typename T>
-VTKM_EXEC_CONT inline T AtomicCompareExchange(
+VTKM_EXEC_CONT inline bool AtomicCompareExchange(
   T* shared,
   T* expected,
   T desired,

vtkm/cont/AtomicArray.h

@@ -26,7 +26,8 @@ namespace cont
 /// \brief A type list containing types that can be used with an AtomicArray.
 ///
 /// @cond NONE
-using AtomicArrayTypeList = vtkm::List<vtkm::UInt32, vtkm::Int32, vtkm::UInt64, vtkm::Int64>;
+using AtomicArrayTypeList =
+  vtkm::List<vtkm::UInt32, vtkm::Int32, vtkm::UInt64, vtkm::Int64, vtkm::Float32, vtkm::Float64>;
 
 struct VTKM_DEPRECATED(1.6,
                        "AtomicArrayTypeListTag replaced by AtomicArrayTypeList. Note that the "

vtkm/cont/arg/testing/UnitTestTypeCheckArray.cxx

@@ -89,7 +89,7 @@ void TestCheckAtomicArray()
                    "Check for 32-bit int failed.");
   VTKM_TEST_ASSERT((TypeCheck<TypeCheckTagAtomicArray, Int64Array>::value),
                    "Check for 64-bit int failed.");
-  VTKM_TEST_ASSERT(!(TypeCheck<TypeCheckTagAtomicArray, FloatArray>::value),
+  VTKM_TEST_ASSERT((TypeCheck<TypeCheckTagAtomicArray, FloatArray>::value),
                    "Check for float failed.");
 }
 

vtkm/exec/AtomicArrayExecutionObject.h

@@ -49,6 +49,32 @@ struct MakeUnsigned<vtkm::Int64>
 {
   using type = vtkm::UInt64;
 };
+template <>
+struct MakeUnsigned<vtkm::Float32>
+{
+  using type = vtkm::UInt32;
+};
+template <>
+struct MakeUnsigned<vtkm::Float64>
+{
+  using type = vtkm::UInt64;
+};
+
+template <typename T>
+struct ArithType
+{
+  using type = typename MakeUnsigned<T>::type;
+};
+template <>
+struct ArithType<vtkm::Float32>
+{
+  using type = vtkm::Float32;
+};
+template <>
+struct ArithType<vtkm::Float64>
+{
+  using type = vtkm::Float64;
+};
 }
 
 template <typename T, typename... MaybeDevice>
@@ -168,7 +194,7 @@ public:
     // This is safe, since the only difference between signed/unsigned types
     // is how overflow works, and signed overflow is already undefined. We also
     // document that overflow is undefined for this operation.
-    using APIType = typename detail::MakeUnsigned<ValueType>::type;
+    using APIType = typename detail::ArithType<ValueType>::type;
 
     return static_cast<T>(
       vtkm::AtomicAdd(reinterpret_cast<APIType*>(this->Data + index), static_cast<APIType>(value)));

vtkmstd/CMakeLists.txt

@@ -10,6 +10,7 @@
 
 set(headers
   aligned_union.h
+  bit_cast.h
   integer_sequence.h
   is_trivial.h
   void_t.h

vtkmstd/bit_cast.h

@@ -0,0 +1,35 @@
+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//============================================================================
+#ifndef vtk_m_std_bit_cast_h
+#define vtk_m_std_bit_cast_h
+
+#include <cstring>
+#include <type_traits>
+
+namespace vtkmstd
+{
+// Copy/Paste from cppreference.com
+template <class To, class From>
+typename std::enable_if<sizeof(To) == sizeof(From) && std::is_trivially_copyable<From>::value &&
+                          std::is_trivially_copyable<To>::value,
+                        To>::type
+// constexpr support needs compiler magic
+bit_cast(const From& src) noexcept
+{
+  static_assert(
+    std::is_trivially_constructible<To>::value,
+    "This implementation additionally requires destination type to be trivially constructible");
+
+  To dst;
+  std::memcpy(&dst, &src, sizeof(To));
+  return dst;
+}
+}
+#endif //vtk_m_std_bit_cast_h