multigpus_culc.jl

# setting process
using Distributed, DistributedArrays 

NUM_GPUS = 4
addprocs(NUM_GPUS)

@everywhere using Distributed, DistributedArrays

# using in all process
@everywhere using CUDAnative, CuArrays

# bind each process with a GPU 
asyncmap(collect(zip(workers(), CUDAnative.devices()))) do (p, d)
  remotecall_wait(() -> CUDAnative.device!(d), p)
  nothing
end

# check
asyncmap(workers()) do w
  remotecall_wait(()->println("process [$(myid())]: use device $(CUDAnative.device())"), w)
end

da = DArray([@spawnat p rand(Float32, 2^20) for p in workers()])
db = DArray([@spawnat p rand(Float32, 2^20) for p in workers()])

cda = DArray([@spawnat p cu(rand(Float32, 2^20)) for p in workers()])
cdb = DArray([@spawnat p cu(rand(Float32, 2^20)) for p in workers()])

using BenchmarkTools

@btime DArray(size($da),procs($da)) do I
  localpart($da) .+ localpart($db)
end

@btime DArray(size($cda),procs($cda)) do I
  localpart($cda) .+ localpart($cdb)
end