overlap copies

Author: agnesLeroy

backends/tfhe-cuda-backend/cuda/src/integer/bitwise_ops.cuh

@@ -53,8 +53,8 @@ __host__ void host_integer_radix_bitop_kb(
   }
 
   integer_radix_apply_bivariate_lookup_table_kb<Torus>(
-      streams, lwe_array_out, lwe_array_1, lwe_array_2,
-      bsks, ksks, ms_noise_reduction_key, lut,
+      streams, lwe_array_out, lwe_array_1, lwe_array_2, bsks, ksks,
+      ms_noise_reduction_key, lut,
       lwe_array_out->num_radix_blocks * lwe_array_out->num_radix_ciphertexts,
       lut->params.message_modulus);
 

tfhe-benchmark/benches/high_level_api/arrays.rs

@@ -1,13 +1,11 @@
+use benchmark::params_aliases::BENCH_PARAM_GPU_MULTI_BIT_GROUP_4_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128;
 use benchmark::utilities::{write_to_json, OperatorType};
-use criterion::{Criterion};
+use criterion::Criterion;
 use rand::prelude::*;
-use benchmark::params_aliases::BENCH_PARAM_GPU_MULTI_BIT_GROUP_4_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128;
+use tfhe::array::GpuFheUint64Array;
 use tfhe::keycache::NamedParam;
 use tfhe::prelude::*;
-use tfhe::{
-    ClientKey, CompressedServerKey,
-};
-use tfhe::array::GpuFheUint64Array;
+use tfhe::{ClientKey, CompressedServerKey};
 
 #[cfg(feature = "gpu")]
 fn main() {
@@ -16,7 +14,7 @@ fn main() {
         let config = ConfigBuilder::with_custom_parameters(
             BENCH_PARAM_GPU_MULTI_BIT_GROUP_4_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M128,
         )
-            .build();
+        .build();
         let cks = ClientKey::generate(config);
         let compressed_sks = CompressedServerKey::new(&cks);
 
@@ -27,16 +25,23 @@ fn main() {
     let array_dim = 32;
     let num_elems = array_dim * array_dim;
     let mut rng = thread_rng();
-    let clear_xs = (0..num_elems as u64).map(| _ | rng.gen::<u64>()).collect::<Vec<_>>();
-    let clear_ys = (0..num_elems as u64).map(| _ | rng.gen::<u64>()).collect::<Vec<_>>();
-
-    let xs = GpuFheUint64Array::try_encrypt((clear_xs.as_slice(), vec![array_dim, array_dim]), &cks).unwrap();
-    let ys = GpuFheUint64Array::try_encrypt((clear_ys.as_slice(), vec![array_dim, array_dim]), &cks).unwrap();
+    let clear_xs = (0..num_elems as u64)
+        .map(|_| rng.gen::<u64>())
+        .collect::<Vec<_>>();
+    let clear_ys = (0..num_elems as u64)
+        .map(|_| rng.gen::<u64>())
+        .collect::<Vec<_>>();
+
+    let xs =
+        GpuFheUint64Array::try_encrypt((clear_xs.as_slice(), vec![array_dim, array_dim]), &cks)
+            .unwrap();
+    let ys =
+        GpuFheUint64Array::try_encrypt((clear_ys.as_slice(), vec![array_dim, array_dim]), &cks)
+            .unwrap();
 
     let mut c = Criterion::default().configure_from_args();
     let bench_id = format!("bench::hlapi::array::cuda::bitand::");
     c.bench_function(&bench_id, |b| {
-
         b.iter(|| {
             let _ = &xs & &ys;
         })
@@ -54,6 +59,5 @@ fn main() {
         vec![],
     );
 
-
     c.final_summary();
 }

tfhe/src/core_crypto/gpu/entities/lwe_ciphertext_list.rs

@@ -104,12 +104,19 @@ impl<T: UnsignedInteger> CudaLweCiphertextList<T> {
                 .map(|list| list.0.lwe_ciphertext_count.0)
                 .sum(),
         );
-
         assert_ne!(
             lwe_ciphertext_count.0, 0,
             "Empty iterator of CudaLweCiphertextList"
         );
 
+        let stream_count = lwe_ciphertext_count.0.max(6);
+        let mut new_streams: Vec<CudaStreams> = Vec::with_capacity(stream_count);
+
+        for _ in 0..stream_count {
+            let stream = CudaStreams::new_single_gpu(streams.gpu_indexes[0]);
+            new_streams.push(stream);
+        }
+
         let first_item = cuda_ciphertexts_list_vec.next().unwrap();
         let lwe_dimension = first_item.lwe_dimension();
         let mut d_vec = CudaVec::new(
@@ -123,25 +130,20 @@ impl<T: UnsignedInteger> CudaLweCiphertextList<T> {
             * std::mem::size_of::<T>();
         // Concatenate gpu_index memory
         unsafe {
-            cuda_memcpy_async_gpu_to_gpu(
-                ptr,
-                first_item.0.d_vec.as_c_ptr(0),
-                size as u64,
-                streams.ptr[0],
-                streams.gpu_indexes[0].get(),
-            );
-            ptr = ptr.wrapping_byte_add(size);
-            for list in cuda_ciphertexts_list_vec {
+            for (i, list) in cuda_ciphertexts_list_vec.enumerate() {
                 cuda_memcpy_async_gpu_to_gpu(
                     ptr,
                     list.0.d_vec.as_c_ptr(0),
                     size as u64,
-                    streams.ptr[0],
-                    streams.gpu_indexes[0].get(),
+                    new_streams[i % stream_count].ptr[0],
+                    new_streams[i % stream_count].gpu_indexes[0].get(),
                 );
                 ptr = ptr.wrapping_byte_add(size);
             }
         }
+        for s in new_streams.iter() {
+            s.synchronize();
+        }
 
         let cuda_lwe_list = CudaLweList {
             d_vec,

tfhe/src/integer/gpu/mod.rs

@@ -7761,10 +7761,7 @@ pub unsafe fn unchecked_bitop_vec_radix_kb_assign<T: UnsignedInteger, B: Numeric
         keyswitch_key.ptr.as_ptr(),
         &raw const ms_noise_reduction_key_ffi,
     );
-    cleanup_cuda_integer_bitop(
-        streams.ffi(),
-        std::ptr::addr_of_mut!(mem_ptr),
-    );
+    cleanup_cuda_integer_bitop(streams.ffi(), std::ptr::addr_of_mut!(mem_ptr));
     update_noise_degree(radix_lwe_left, &cuda_ffi_radix_lwe_left);
     streams.synchronize();
 }