ZMQ Multiple Publisher and Single Subscriber -- data loss observed - zeromq

Question

I have created 2 Publishers connecting to the same static location .
Publisher1
dummyFrontEnd::dummyFrontEnd():context(1),socket(context,ZMQ_PUB) {
}
void dummyFrontEnd::Init()
{
socket.connect("tcp://127.0.0.1:5555");
cout << "Connecting .... " << endl;
}
void dummyFrontEnd::SendTwoTables()
{
cout << "In SendTwoTables" <<endl;
while(1) {
canlogreq canLogObj = canlogreq::default_instance();
canLogObj.set_fromhours(11);
canLogObj.set_fromminutes(7);
canLogObj.set_fromseconds(2);
canLogObj.set_fromday(16);
canLogObj.set_frommonth(5);
canLogObj.set_fromyear(2020);
canLogObj.set_tohours(12);
canLogObj.set_tominutes(7);
canLogObj.set_toseconds(4);
canLogObj.set_today(17);
canLogObj.set_tomonth(5);
canLogObj.set_toyear(2020);
zmq::message_t logsnippetmsg(canLogObj.ByteSizeLong() + sizeof(uint16_t));
*((uint16_t*)logsnippetmsg.data()) = 20;
canLogObj.SerializeToArray(logsnippetmsg.data()+sizeof(uint16_t), canLogObj.ByteSizeLong());
socket.send(logsnippetmsg);
usleep(1);
canLogObj.clear_fromhours();
canLogObj.clear_fromminutes();
canLogObj.clear_fromseconds();
canLogObj.clear_fromday();
canLogObj.clear_frommonth();
canLogObj.clear_fromyear();
canLogObj.clear_tohours();
canLogObj.clear_tominutes();
canLogObj.clear_toseconds();
canLogObj.clear_today();
canLogObj.clear_tomonth();
canLogObj.clear_toyear();
}
}
Publisher2:
dummyFrontEnd::dummyFrontEnd():context(1),socket(context,ZMQ_PUB) {
}
void dummyFrontEnd::Init()
{
socket.connect("tcp://127.0.0.1:5555");
cout << "Connecting .... " << endl;
}
void dummyFrontEnd:: SendData() {
while (std::getline(file, line_str)) {
std::stringstream ss(line_str);
double tdiff;
int i;
char J;
int _1939;
int pgn;
char p;
int priority;
char _0;
int source;
char dash;
std::string direction;
char d;
int length;
int data[8];
ss >> tdiff >> i >> J >> _1939 >> pgn >> p >> priority >> _0 >> source
>> dash >> direction >> d >> length >> data[0] >> data[1] >> data[2]
>> data[3] >> data[4] >> data[5] >> data[6] >> data[7];
timestamp += tdiff;
while (gcl_get_time_ms() - start_time <
uint64_t(timestamp * 1000.0) - first_time) { usleep(1); }
if (arguments.verbose) {
std::cout << timestamp << " " << i << " " << J << " " << _1939 << " "
<< pgn << " " << p << " " << priority << " " << _0 << " " << source
<< " " << dash << " " << direction << " " << d << " " << length
<< " " << data[0] << " " << data[1] << " " << data[2] << " "
<< data[3] << " " << data[4] << " " << data[5] << " " << data[6]
<< " " << data[7] << std::endl;
}
uint64_t timestamp_ms = (uint64_t)(timestamp * 1000.0);
protoTable.add_columnvalues(uint64ToString(timestamp_ms) /* timestamp */);
protoTable.add_columnvalues(intToString(pgn) /* PGN */);
protoTable.add_columnvalues(intToString(priority) /* Priority */);
protoTable.add_columnvalues(intToString(source) /* Source */);
protoTable.add_columnvalues(direction /* Direction */);
protoTable.add_columnvalues(intToString(length) /* Length */);
protoTable.add_columnvalues(intToString(data[0]) /* data1 */);
protoTable.add_columnvalues(intToString(data[1]) /* data2 */);
protoTable.add_columnvalues(intToString(data[2]) /* data3 */);
protoTable.add_columnvalues(intToString(data[3]) /* data4 */);
protoTable.add_columnvalues(intToString(data[4]) /* data5 */);
protoTable.add_columnvalues(intToString(data[5]) /* data6 */);
protoTable.add_columnvalues(intToString(data[6]) /* data7 */);
protoTable.add_columnvalues(intToString(data[7]) /* data8 */);
zmq::message_t create_values(protoTable.ByteSizeLong()+sizeof(uint16_t));
*((uint16_t*)create_values.data()) = TABLEMSG_ID; // ID
protoTable.SerializeToArray(create_values.data()+sizeof(uint16_t), protoTable.ByteSizeLong());
socket.send(create_values);
protoTable.clear_columnvalues();
usleep(1);
}
}
Subscriber
TransportLayer::TransportLayer():context(1),socket(context,ZMQ_SUB){ }
void TransportLayer::Init()
{
socket.bind("tcp://*:5555");
socket.setsockopt(ZMQ_SUBSCRIBE, "", 0);
}
void TransportLayer::Receive()
{
cout <<"TransportLayer::Receive " << " I m in server " << endl;
static int count = 1 ;
// Producer thread.
while ( true ){
zmq::message_t request;
string protoBuf;
socket.recv(&request);
uint16_t id = *((uint16_t*)request.data());
cout <<"TransportLayer : "<<"request.data: "<< request.data() << endl;
cout << "TransportLayer: count " << count<< endl; count = count+1 ;
cout <<"TransportLayer : request.data.size "<< request.size() << endl;
protoBuf = std::string(static_cast<char*>(request.data()+sizeof(uint16_t)), request.size()-sizeof(uint16_t));
cout << "ProtoBuf : " << protoBuf << endl;
InterfaceLayer *interfaceLayObj = InterfaceLayer::getInstance();
switch(id) {
case TABLEMSG_ID: cout << "Canlyser" << endl;
interfaceLayObj->ParseProtoBufTable(protoBuf);
break;
case LOGSNIPPET_ID:cout << "LogSnip" << endl;
interfaceLayObj->ParseProtoBufLogSnippet(protoBuf);
interfaceLayObj->logsnippetSignal(); // publish the signal
break;
default:
break;
}
usleep(1);
}
}
Observation :
I)
Execution Order .
1. Started Subscriber
2. Started Publisher1 ( it sent only one data value)
Subscriber missed to receive this data.
II) modified Publisher1 to send the same data in a while loop
Execution Order
1. Started Subscriber
2. Started Publisher1
3. Started Publsiher2 .
Now I see that Subscriber is receiving the data from both publishers .
This gives me an indication that there is a possibility for data loss.
How do I ensure there is absolutely no data loss.
Thanks

Answer 1

Your producer may be sending its one message before the subscription handshaking has completed. As a result, the publish socket discards the message because there is no subscription registered.
If you use an XPUB socket (ZMQ_XPUB -- see http://api.zeromq.org/4-2:zmq-socket) instead of a PUB socket, your program can wait for a subscribe message, so that it knows that someone is listening, before sending its message(s).