Julia: How to copy data to another processor in Julia - parallel-processing

Question

How do you move data from one processor to another in julia?
Say I have an array
a = [1:10]
Or some other data structure. What is the proper way to put it on all other available processors so that it will be available on those processors as the same variable name?

Answer 1

I didn't know how to do this at first, so I spent some time figuring it out.
Here are some functions I wrote to pass objects:
sendto
Send an arbitrary number of variables to specified processes.
New variables are created in the Main module on specified processes. The
name will be the key of the keyword argument and the value will be the
associated value.
function sendto(p::Int; args...)
for (nm, val) in args
#spawnat(p, eval(Main, Expr(:(=), nm, val)))
end
end
function sendto(ps::Vector{Int}; args...)
for p in ps
sendto(p; args...)
end
end
Examples
# creates an integer x and Matrix y on processes 1 and 2
sendto([1, 2], x=100, y=rand(2, 3))
# create a variable here, then send it everywhere else
z = randn(10, 10); sendto(workers(), z=z)
getfrom
Retrieve an object defined in an arbitrary module on an arbitrary
process. Defaults to the Main module.
The name of the object to be retrieved should be a symbol.
getfrom(p::Int, nm::Symbol; mod=Main) = fetch(#spawnat(p, getfield(mod, nm)))
Examples
# get an object from named x from Main module on process 2. Name it x
x = getfrom(2, :x)
passobj
Pass an arbitrary number of objects from one process to arbitrary
processes. The variable must be defined in the from_mod module of the
src process and will be copied under the same name to the to_mod
module on each target process.
function passobj(src::Int, target::Vector{Int}, nm::Symbol;
from_mod=Main, to_mod=Main)
r = RemoteRef(src)
#spawnat(src, put!(r, getfield(from_mod, nm)))
for to in target
#spawnat(to, eval(to_mod, Expr(:(=), nm, fetch(r))))
end
nothing
end
function passobj(src::Int, target::Int, nm::Symbol; from_mod=Main, to_mod=Main)
passobj(src, [target], nm; from_mod=from_mod, to_mod=to_mod)
end
function passobj(src::Int, target, nms::Vector{Symbol};
from_mod=Main, to_mod=Main)
for nm in nms
passobj(src, target, nm; from_mod=from_mod, to_mod=to_mod)
end
end
Examples
# pass variable named x from process 2 to all other processes
passobj(2, filter(x->x!=2, procs()), :x)
# pass variables t, u, v from process 3 to process 1
passobj(3, 1, [:t, :u, :v])
# Pass a variable from the `Foo` module on process 1 to Main on workers
passobj(1, workers(), [:foo]; from_mod=Foo)

Answer 2

use #eval #everywhere... and escape the local variable. like this:
julia> a=collect(1:3)
3-element Array{Int64,1}:
1
2
3
julia> addprocs(1)
1-element Array{Int64,1}:
2
julia> #eval #everywhere a=$a
julia> #fetchfrom 2 a
3-element Array{Int64,1}:
1
2
3

Answer 3

Just so everyone here knows, I put these ideas together into a package ParallelDataTransfer.jl for this. So you just need to do
using ParallelDataTransfer
(after installing) in order to use the functions mentioned in the answers here. Why? These functions are pretty useful! I added some testing, some new macros, and updated them a bit (they pass on v0.5, fail on v0.4.x). Feel free to put in pull requests to edit these and add more.

Answer 4

To supplement #spencerlyon2 's answer here are some macros:
function sendtosimple(p::Int, nm, val)
ref = #spawnat(p, eval(Main, Expr(:(=), nm, val)))
end
macro sendto(p, nm, val)
return :( sendtosimple($p, $nm, $val) )
end
macro broadcast(nm, val)
quote
#sync for p in workers()
#async sendtosimple(p, $nm, $val)
end
end
end
The #spawnat macro binds a value to a symbol on a particular process
julia> #sendto 2 :bip pi/3
RemoteRef{Channel{Any}}(9,1,5340)
julia> #fetchfrom 2 bip
1.0471975511965976
The #broadcast macro binds a value to a symbol in all processes except 1 (as I found doing so made future expressions using the name copy the version from process 1)
julia> #broadcast :bozo 5
julia> #fetchfrom 2 bozo
5
julia> bozo
ERROR: UndefVarError: bozo not defined
julia> bozo = 3 #these three lines are why I exclude pid 1
3
julia> #fetchfrom 7 bozo
3
julia> #fetchfrom 7 Main.bozo
5