Error in Prosses <proc_num> with exit value: - parallel-processing

I'm trying to parallelize a function but I seem to be having problems (in Erlang).
I have code similar to this:
-module(examples).
-import(lists, [sublist/2, sublist/3]).
motherFunc(Ls) -> childFunc(Ls, false).
childFunc(Ls, false) ->
Pid1 = spawn(example, mergesortP, [lists:sublist(Ls, length(Ls) div 2, self()]),
Pid2 = spawn(example, mergesortP, [lists:sublist(Ls, length(Ls) div 2, self()]),
FirstHalf = [],
SecondHalf = [],
receive
{Pid1, Msg} -> FirstHalf = Msg;
{Pid2, Msg} -> SecondHalf = Msg
end,
SecondHalf ++ FirstHalf;
childFunc([], Pid) -> Pid ! {self(), []};
childFunc([L], Pid) -> Pid ! {self(), [L]};
childFunc(Ls, Pid) ->
Pid1 = spawn(examples, childFunc, [lists:sublist(Ls, length(Ls) div 2, self()]),
Pid2 = spawn(examples, childFunc, [lists:sublist(Ls, length(Ls) div 2 + 1, length(Ls) div 2 + 1), self()]),
FirstHalf = [],
SecondHalf = [],
receive
{Pid1, Msg} -> FirstHalf = Msg;
{Pid2, Msg} -> SecondHalf = Msg
end,
Pid ! {self(), SecondHalf ++ FirstHalf}.
When I run this I'm getting this error message for both threads, and then it does nothing else: Error in process <0.31.0> with exit value: {undef,[{examples,childFunc,[[1,2,3,4],<0.2.0>],[]}]}

The function spawn/3 with the module and the function name as the first and second argument requires that the function is exported.
In fact, any explicit qualified call:
module:function(...)
or implicit qualified call:
apply(module, function, ...)
requires that the function be exported. Whether it's supposedly from the same module or not is irrelevant (1).
In your code, you should either use spawn/1 or export the function:
-export([childFunc/2, mergesortP/2]).
% ...
spawn(?MODULE, mergesortP, [lists:sublist(Ls, length(Ls) div 2), self()])
or
Self = self(),
spawn(fun() -> mergesortP(lists:sublist(Ls, length(Ls) div 2), Self) end.
Note the difference when passing self(). With spawn/3 invocation, self() is evaluated by the spawning process and therefore is its pid. If you call self() within the anonymous function passed to spawn/1, it will evaluate to the pid of the spawned process. So you need to call it before and pass it in a variable.
Additionally, your code is unlikely to work. Indeed, the following section does not do what you mean:
FirstHalf = [],
SecondHalf = [],
receive
{Pid1, Msg} -> FirstHalf = Msg;
{Pid2, Msg} -> SecondHalf = Msg
end,
This will first assign [] to FirstHalf and SecondHalf variables, and then try to match this with the first message that arrived, as variables in Erlang cannot be re-assigned. The second message is ignored as received is only used once. The first result probably isn't an empty list and therefore it will fail with a badmatch error. Instead, it seems you mean to perform parallel execution of the merge an collect the results afterwards. You could write:
FirstHalf = receive {Pid1, Msg1} -> Msg1 end,
SecondHalf = receive {Pid2, Msg2} -> Msg2 end,
This will collect the first half and then the second half.
(1) Actually, a qualified call can in fact be executed by another module, or more precisely a newer version of the module. Qualifying calls to (a loop) function within the same module is a common method to continue execution in a newer version during code upgrades.

Related

How can I wait for multiple child processes using the OCaml Unix module?

I'm trying to parallelize work. I'm trying to have subprocesses each take a piece of the work, then wait for the work to complete from the parent process.
The following solution seems to work for small examples:
let par_iter (items: 'i list) ~(f: 'i -> unit): unit =
let orig_pid = Unix.getpid () in
let rec loop = function
| [] -> ()
| h :: t ->
match Unix.fork () with
| 0 -> f h
| _ -> loop t
in
loop items;
if Unix.getpid () = orig_pid then
List.iter items ~f:(fun _ -> ignore ## Unix.wait ())
else
exit 0
Usage:
let () =
par_iter [1; 2; 3; 4] ~f:do_something_with_an_int;
The contract of Unix.wait when there are multiple subprocesses is not very clear to me. Here I'm relying on the behavior where waiting n times will wait for all and only n subprocesses to finish.
Does this solution rely on undefined behavior? Is there an alternative way to correctly implement par_iter such that it spawns work on child processes and waits for the work to finish?
tested on OCaml 4.14.0
If the original parent process already had some subprocesses, this can fail. Thus, IMHO it's not usable as a general library. You should use Unix.waitpid to wait specifically for the processes you created.

How to call using gen_event in Erlang?

I am using a gen_event behaviour and when i am trying to issue a gen_event:call i get the following error:
> =CRASH REPORT==== 22-Dec-2019::19:17:43.030000 === crasher:
> initial call: gen_event:init_it/6
> pid: <0.215.0>
> registered_name: hev
> exception exit: {undef,[{fm,state,[<0.215.0>],[]},
> {erl_eval,do_apply,6,
> [{file,"erl_eval.erl"},{line,684}]},
> {shell,exprs,7,[{file,"shell.erl"},{line,686}]},
> {shell,eval_exprs,7,
> [{file,"shell.erl"},{line,642}]},
> {shell,eval_loop,3,
> [{file,"shell.erl"},{line,627}]}]}
> in function gen_event:terminate_server/4 (gen_event.erl, line 354)
> ancestors: [<0.212.0>]
> message_queue_len: 1
> messages: [{'EXIT',<0.212.0>,normal}]
> links: []
> dictionary: []
> trap_exit: true
> status: running
> heap_size: 610
> stack_size: 27
> reductions: 279 neighbours:
My event manager and event handler get spawned and i can successfully issue notify (i get ok back) but i can not call:
Module
-module(hev).
-export([start/0,append/2,state/1]).
-export([init/1,terminate/2,code_change/3,handle_call/2,handle_event/2]).
-record(state,{
xs=[]
}).
-behaviour(gen_event).
%callbacks
init([])->
{ok,#state{xs=[1]}}.
**API**
start()->
{ok,Pid}=gen_event:start_link({local,?MODULE}),
gen_event:add_handler(Pid,some_handler,[]),
Pid.
append(Pid,Elem)->
gen_event:notify(Pid,{append,Elem}).
state(Pid)->
gen_event:call(Pid,state).
Handlers
handle_event({append,Elem},State=#state{xs=XS})->
{ok,#state{xs=[Elem|XS]}};
handle_call(state,State})->
{ok,State,State};
handle_call(Event,State)->
{ok,nada_for_you,State}.
P.S I have not posted all the requried methods (code_change,terminate..etc) but they exist.
I have not posted all the requried methods (code_change,terminate..etc) but they exist.
1) They are optional anyway. Check the big green Notes in the docs, e.g. terminate().
2) As for your error message:
pid: <0.215.0>
registered_name: hev
exception exit: {undef,[{fm,state,[<0.215.0>],[]},
it seems to be saying that there was a process (with pid=<0.215.0>), which was registered with the name hev, that tried to execute a function named fm:state() with one argument but there was no function fm:state/1 defined anywhere, hence the undef exception. Because you posted no module named fm, the error makes no sense in relation to the code you posted.
3) Your code also specifies a module named some_handler that does not exist:
gen_event:add_handler(Pid,some_handler,[]),
4) You have a basic syntax error here:
handle_call(state,State})->
5) You are calling a couple of functions with the wrong number of arguments.
You need to be more diligent about posting code that will actually produce the error you experienced.
Here's a simple example using gen_event to create a counter:
-module(counter).
-behaviour(gen_event).
-compile(export_all).
%% Callback functions:
init(StartingCount) -> % Called by gen_event:add_handler()
State = StartingCount,
{ok, State}.
terminate(_Reason, State) ->
io:format("Terminating state was: ~w~n", [State]).
% Calls to gen_event:notify() cause this function to execute:
handle_event({increase, Change}, State) ->
NewState = State+Change,
{ok, NewState};
handle_event({decrease, Change}, State) ->
NewState = State-Change,
{ok, NewState}.
% Calls to gen_event:call() cause this function to execute:
handle_call(get_count, State) ->
Reply = io_lib:format("Reply from handle_call(): count is ~w~n", [State]),
{ok, Reply, State};
handle_call({increase_and_get_count, Change}, State) ->
NewState = State+Change,
Reply = io_lib:format("Reply from handle_call(): count is ~w~n", [NewState]),
{ok, Reply, NewState}.
%% User interface functions:
start(StartingCount) ->
ServerName = gen_event_counter,
CallbackModule = counter,
{ok, _Pid} = gen_event:start_link({local, ServerName}),
%Name of process running gen_event server is: gen_event_counter
ok = gen_event:add_handler(ServerName, CallbackModule, StartingCount).
%StartingCount is passed to init() callback
stop() ->
ok = gen_event:stop(gen_event_counter),
stopped.
send_request_with_notify(Request) ->
gen_event:notify(gen_event_counter, Request). % returns immediately, does not wait for a reply.
% Request = {increase, 1}, {decrease, 2}, etc.
% Request is passed as first arg to handle_event().
send_request_with_call(Request) ->
Reply = gen_event:call(gen_event_counter, counter, Request), % waits for a reply
% Request is passed as first arg to handle_call()
io:format("send_request_with_call() returned => ~s", [Reply]).
In the shell:
~/erlang_programs/gen_event$ erl
Erlang/OTP 20 [erts-9.3] [source] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] [hipe] [kernel-poll:false]
Eshell V9.3 (abort with ^G)
1> c(counter).
counter.erl:3: Warning: export_all flag enabled - all functions will be exported
{ok,counter}
2> counter:start(0).
ok
3> counter:send_request_with_call(get_count).
send_request_with_call() returned => Reply from handle_call(): count is 0
ok
4> counter:send_request_with_notify({increase, 2}).
ok
5> counter:send_request_with_call(get_count).
send_request_with_call() returned => Reply from handle_call(): count is 2
ok
6> counter:send_request_with_call({increase_and_get_count, 5}).
send_request_with_call() returned => Reply from handle_call(): count is 7
ok
7> counter:stop().
Terminating state was: 7
stopped
8>
After fixing all the errors in your code:
-module(hev).
-compile(export_all).
-behaviour(gen_event).
-record(state,{
xs=[]
}).
%callbacks
init(no_args)->
{ok, #state{xs=[1]} }.
handle_event({append,Elem}, #state{xs=XS} ) ->
io:format("hev:handle_event() called~n"),
{ok, #state{xs=[Elem|XS]}}.
handle_call(get_state, State)->
Reply = State,
{ok, Reply, State};
handle_call(_Other, State)->
Reply = nada_for_you,
{ok, Reply, State}.
%**API**
start()->
gen_event:start_link({local, ?MODULE}), %Sets the gen_event server name to ?MODULE
% Server Callback Args for
% Name module init()
gen_event:add_handler(?MODULE, ?MODULE, no_args).
%Tells the gen_event server named ?MODULE to look for the callback functions
%in a module also named ?MODULE
append(Elem)->
% Server Request
% Name (matches against 1st arg in handle_event() )
gen_event:notify(?MODULE, {append, Elem}).
get_state()->
% Server Calback Request
% Name module (matches against 1st arg in handle_call() )
gen_event:call(?MODULE, ?MODULE, get_state).
other_calls() ->
gen_event:call(?MODULE, ?MODULE, {set_state, [1, 2, 3]}).
stop() ->
ok = gen_event:stop(?MODULE),
stopped.
In the shell:
~/erlang_programs/gen_event$ erl
Erlang/OTP 20 [erts-9.3] [source] [64-bit] [smp:4:4] [ds:4:4:10] [async-threads:10] [hipe] [kernel-poll:false]
Eshell V9.3 (abort with ^G)
1> c(hev).
hev.erl:2: Warning: export_all flag enabled - all functions will be exported
{ok,hev}
2> hev:start().
ok
3> hev:get_state().
{state,[1]}
4> hev:append(45).
ok
hev:handle_event() called
5> hev:get_state().
{state,[45,1]}
6> hev:other_calls().
nada_for_you
7> hev:stop().
stopped
8>
Note that notify() causes the handle_event() function in all the modules that were added with add_handler() to execute, whereas call() targets a specific module's handle_call() function.

How can I factor multiples function that gets their result process by another function?

I would like to factorize this code :
(* This function is applied to the result of functions below *)
let manage_result r s =
match r with
| Error ( `Msg e ) -> Tsdl.Sdl.log s e;exit 1
| Ok a -> a
(* Examples of function I want to factorize, let's call them f_functions, f for factorize *)
let init () =
let flag = Tsdl.Sdl.Init.everything in
let result = Tsdl.Sdl.init flag in
manage_result result "Init error : %s"
let create_window title w h =
let flag = Tsdl.Sdl.Window.windowed in
let result = Tsdl.Sdl.create_window title ~w:w ~h:h flag in
manage_result result "Create window error : %s"
let get_window_surface window =
let result = Tsdl.Sdl.get_window_surface window in
manage_result result "Get window surface error : %s"
As you can see, the two last lines of all of these f_functions are very similar. I would like to make a function that takes as argument a function ( for example, if I wanted to factorize init, the function passed as a parameter would be Tsdl.Sdl.init) and return a function that return the return value of function passed as an argument AND processed through manage_result.
The difficulty is that I don't know how many argument can the f_functions take.
Any other recommendations is appreciated!
Thank you.
A potential solution might be to use the pipe operator rather than naming the intermediary result
let on_error s r = manage_result r s
let create_window title w h =
let flag = Tsdl.Sdl.Window.windowed in
Tsdl.Sdl.create_window title ~w:w ~h:h flag
|> on_error "Create window error : %s"
Going one step further, we could define a custom operator for the error handling
let ( <!> ) = manage_result
which may make your definition lightweight enough
let create_window title w h =
let flag = Tsdl.Sdl.Window.windowed in
Tsdl.Sdl.create_window title ~w:w ~h:h flag
<!> "Create window error : %s"

Buffer size in Erlang / Golang port example

I have a crude Erlang-to-Golang port example, passing data from Erlang to Golang and echoing the response.
Problem is the amount of data I can transfer seems to be limited to 2^8 bytes (see below). I thought the problem was probably on the Golang side (not creating a big enough buffer) but replacing bufio.NewReader with bufio.NewReaderSize didn't work. So am now thinking the problem is maybe on the Erlang side.
What do I need to do to increase the buffer size / be able to echo a message larger than 2^8 bytes ?
TIA
justin#justin-ThinkPad-X240:~/work/erlang_golang_port$ erl -pa ebin
Erlang/OTP 17 [erts-6.4.1] [source] [64-bit] [smp:4:4] [async-threads:10] [kernel-poll:false]
Eshell V6.4.1 (abort with ^G)
1> port:start("./echo").
<0.35.0>
2> port:ping(65000).
65000
3> port:ping(66000).
** exception error: bad argument
in function port:call_port/1 (port.erl, line 20)
4> port:start("./echo").
<0.40.0>
5> port:ping(66000).
65536
Go
package main
import (
"bufio"
"os"
)
const Delimiter = '\n'
func main() {
// reader := bufio:NewReader(os.Stdin)
reader := bufio.NewReaderSize(os.Stdin, 1677216) // 2**24;
bytes, _ := reader.ReadBytes(Delimiter)
os.Stdout.Write(bytes[:len(bytes)-1])
}
Erlang
-module(port).
-export([start/1, stop/0, init/1]).
-export([ping/1]).
-define(DELIMITER, [10]).
start(ExtPrg) ->
spawn(?MODULE, init, [ExtPrg]).
stop() ->
myname ! stop.
ping(N) ->
Msg=[round(65+26*random:uniform()) || _ <- lists:seq(1, N)],
call_port(Msg).
call_port(Msg) ->
myname ! {call, self(), Msg},
receive
{myname, Result} ->
length(Result)
end.
init(ExtPrg) ->
register(myname, self()),
process_flag(trap_exit, true),
Port = open_port({spawn, ExtPrg}, []),
loop(Port).
loop(Port) ->
receive
{call, Caller, Msg} ->
Port ! {self(), {command, Msg++?DELIMITER}},
receive
{Port, {data, Data}} ->
Caller ! {myname, Data}
end,
loop(Port);
stop ->
Port ! {self(), close},
receive
{Port, closed} ->
exit(normal)
end;
{'EXIT', Port, _Reason} ->
exit(port_terminated)
end.
If you use start_link instead, you'll see that the port crashes after the first command:
1> port:start('go run port.go').
<0.118.0>
2> port:ping(65000).
65000
** exception error: port_terminated
If you change the Go code to run in a loop, this crash can be avoided:
func main() {
for {
// reader := bufio:NewReader(os.Stdin)
reader := bufio.NewReaderSize(os.Stdin, 1677216) // 2**24;
bytes, _ := reader.ReadBytes(Delimiter)
os.Stdout.Write(bytes[:len(bytes)-1])
}
}
Now we can see another interesting result:
33> c(port).
{ok,port}
40> port:ping(66000).
65536
41> port:ping(66000).
464
42> port:ping(66000).
65536
43> port:ping(66000).
464
Now we can see that no data is actually lost, it's just buffered in the port. Since you have not specified a framing protocol (using {packet, N} or {line, N} you are responsible yourself for collecting the data. It also seems that the internal buffer size of an Erlang port is 64K (although I found no documentation of this and no way to change it).
If you change your receive to get all data before returning, you'll every byte each time:
loop(Port) ->
receive
{call, Caller, Msg} ->
Port ! {self(), {command, Msg++?DELIMITER}},
Caller ! {myname, receive_all(Port, 10)},
loop(Port);
stop ->
Port ! {self(), close},
receive
{Port, closed} ->
exit(normal)
end;
{'EXIT', Port, _Reason} ->
exit(port_terminated)
end.
receive_all(Port, Timeout) -> receive_all(Port, Timeout, []).
receive_all(Port, Timeout, Data) ->
receive
{Port, {data, New}} ->
receive_all(Port, Timeout, [New|Data])
after Timeout ->
lists:flatten(lists:reverse(Data))
end.
Running this, we get:
1> c(port).
{ok,port}
2>
3> port:start('go run port.go').
<0.311.0>
4> port:ping(66000).
66000
5> port:ping(66000).
66000
6> port:ping(66000).
66000
2^8 is 256, not 65536 which is 2^16 (or 2 bytes).
For excluding golang program you can simply replace your echo with GNU cat
Default message max size for port communication is 64k, so when your port receives messages, the first one is leading 64k of the string. You can read port again to gain remaining data but you just drop them in your code.
If you really want to communicate on line-based protocol you should configure your port accordingly:
{line, L}
Messages are delivered on a per line basis. Each line
(delimited by the OS-dependent newline sequence) is delivered in one
single message. The message data format is {Flag, Line}, where Flag is
either eol or noeol and Line is the actual data delivered (without the
newline sequence).
L specifies the maximum line length in bytes. Lines longer than this
will be delivered in more than one message, with the Flag set to noeol
for all but the last message. If end of file is encountered anywhere
else than immediately following a newline sequence, the last line will
also be delivered with the Flag set to noeol. In all other cases,
lines are delivered with Flag set to eol.
The {packet, N} and {line, L} settings are mutually exclusive.
So your code would be
Port = open_port({spawn, ExtPrg}, [{line, ?PACKET_SIZE]),
%%...
{call, Caller, Msg} ->
Port ! {self(), {command, Msg++?DELIMITER}},
D = read_data(Port, []),
Caller ! {myname, D},
loop(Port);
%%...
read_data(Port, Prefix) ->
receive
{Port, {data, {noeol, Data}}} ->
read_data(Port, Prefix ++ Data);
{Port, {data, {eol, Data}}} ->
Prefix ++ Data
end.
I have been struggling with the similar problem.
Here the complete code of pipe module.
It allows sent text data to port and read all replies.
-module(apr_pipe).
-export([open_pipe/2,send/2,close/1]).
-export([loop/1,status/1,init/1]).
-include_lib("kernel/include/logger.hrl").
-define(MAX_LINE_LEN,4096).
open_pipe(Path,Cmd) ->
State = #{path => Path, cmd => Cmd},
Pid = spawn(?MODULE,init,[State]),
Pid.
init(State) ->
#{path := Path,cmd := Cmd} = State,
FullFn = filename:join(Path,Cmd),
Settings = [{line,?MAX_LINE_LEN},use_stdio,stderr_to_stdout,hide,binary,exit_status],
Port = erlang:open_port({spawn_executable,FullFn},Settings),
State2 = State#{port => Port, data => #{}},
loop(State2).
send(Pid,Data) -> Pid!{self(),send,Data}.
close(Pid) -> Pid!{self(),send,close}.
status(Pid) -> Pid!{self(),status}.
get_eol() -> <<"\n">>.
loop(State) ->
receive
{_Pid,send,close} ->
?LOG(notice,"got cmd: Close",[]),
Port = maps:get(port,State),
port_close(Port),
exit(normal);
{Pid,send,Data} ->
?LOG(notice,"Send Data ...",[]),
Port = maps:get(port,State),
port_command(Port,Data),
port_command(Port,get_eol()),
State2 = State#{status => data_sent, client => Pid},
loop(State2);
{Pid,status} ->
Port = maps:get(port,State),
?LOG(notice,"Status: Port: ~p State: ~p",[Port,State]),
Pid!{status,Port,State},
loop(State);
% port messages.
{Port, {data,{noeol,Data}}} ->
?LOG(notice,"Port: ~p Data: ~p",[Port,Data]),
CurData = maps:get(cur_data,State,[]),
State2 = State#{cur_data => [Data | CurData]},
loop(State2);
{Port, {data, {eol,Data}}} ->
?LOG(notice,"Port: ~p Data: ~p",[Port,Data]),
CurData = [Data | maps:get(cur_data,State,[])],
CurData2 = lists:reverse(CurData),
Reply = list_to_binary(CurData2),
Client = maps:get(client,State,undefined),
State2 = State#{cur_data => [], client => undefined},
case Client of
undefined -> ?LOG(error,"can not sent reply. Client: ~p Reply: ~p", [Client,Reply]),
loop(State2);
_ -> Client!{reply,Reply},
loop(State2)
end;
{_Port, closed} ->
?LOG(warning, "Port: ~p closed",[]),
exit(normal);
{'EXIT', Port, Reason} ->
?LOG(notice,"Port: ~p exit. Reason: ~p",[Port,Reason]),
exit(Reason);
_Other -> ?LOG(error,"unexpected message: ~p",[_Other]),
exit({error,{unexpected_message,_Other}})
end.

RxJs - why Rx.Observable.fromNodeCallack(...)(...).retry() does not retry on error?

I was wondering why the following code (in coffeescript) will not retry as expected.
Rx = require 'rx'
count = 0
functToTest = (cb) ->
console.log "count is", count
count++
if count is 1
cb(new Error('some error'))
else if count is 2
cb(null,2)
else if count is 3
cb(null,3)
else
cb(null,4)
source = Rx.Observable.fromNodeCallback(functToTest)()
onNext = (value) ->
console.log value
onError = (err) ->
console.log err
onCompleted = ->
console.log "done"
retryableSrc = source.retry(3)
retryableSrc.subscribe(onNext, onError, onCompleted)
It will output following messages and quit
count is 0
[Error: some error]
I had thought this is might because fromNodeCallback() return a hot observable. But a test as below show it is NOT.
Rx = require 'rx'
count = 0
functToTest = (cb) ->
console.log "count is", count
count++
if count is 1
cb(new Error('some error'))
else if count is 2
cb(null,2)
else if count is 3
cb(null,3)
else
cb(null,4)
source = Rx.Observable.fromNodeCallback(functToTest)()
onNext = (value) ->
console.log value
onError = (err) ->
console.log err
onCompleted = ->
console.log "done"
retryableSrc = source.retry(3)
setTimeout ( -> ), 1000
If it was a hot observable, the program above should have printed some "count is 0" message. But in reality the program just waits 1 second and quits.
It actually is hot, or goes hot when you first subscribe to it.
Inside of fromNodeCallback is Rx.Observable.create(...).publishLast().refCount() meaning that when you first subscribe it will execute the method, print count then emit an error. The error will be caught downstream by retry, which will resubscribe thrice only to received the cached error, which it will finally emit itself.
You can fix it by using flatMap
ncb = Rx.Observable.fromNodeCallback(functToTest);
source = Rx.Observable.just(ncb).flatMap((fn) -> fn());

Resources