[gloo] Enable using c10::Half for gloo cuda

[ghstack-poisoned]

Author: fduwjj

gloo/cuda.cu

@@ -398,6 +398,12 @@ DELEGATE_SIMPLE_CUDA_BINARY_OPERATOR(BFloat16, cudaSum, +);
 DELEGATE_SIMPLE_CUDA_BINARY_OPERATOR(BFloat16, cudaProduct, *);
 DELEGATE_SIMPLE_CUDA_BINARY_COMPARE(BFloat16, cudaMin, <);
 DELEGATE_SIMPLE_CUDA_BINARY_COMPARE(BFloat16, cudaMax, >);
+using Half = c10::Half;
+INSTANTIATE_COPY_ASYNC(Half);
+DELEGATE_SIMPLE_CUDA_BINARY_OPERATOR(Half, cudaSum, +);
+DELEGATE_SIMPLE_CUDA_BINARY_OPERATOR(Half, cudaProduct, *);
+DELEGATE_SIMPLE_CUDA_BINARY_COMPARE(Half, cudaMin, <);
+DELEGATE_SIMPLE_CUDA_BINARY_COMPARE(Half, cudaMax, >);
 #endif
 
 } // namespace gloo

gloo/cuda_allreduce_bcube.cc

@@ -516,6 +516,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_allreduce_halving_doubling.cc

@@ -659,6 +659,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_allreduce_local.cc

@@ -78,6 +78,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_allreduce_ring.cc

@@ -190,6 +190,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_allreduce_ring_chunked.cc

@@ -367,6 +367,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_broadcast_one_to_all.cc

@@ -199,6 +199,7 @@ INSTANTIATE_TEMPLATE(float16);
 
 #if GLOO_USE_TORCH_DTYPES
 INSTANTIATE_TEMPLATE(c10::BFloat16);
+INSTANTIATE_TEMPLATE(c10::Half);
 #endif
 
 } // namespace gloo

gloo/cuda_private.h

@@ -22,6 +22,7 @@
 
 #if GLOO_USE_TORCH_DTYPES
 #include <c10/util/BFloat16.h>
+#include <c10/util/Half.h>
 #endif
 
 namespace gloo {