In the socket.io acknowledgement example we see a client's send/emit being called back with the server's response. Is the same functionality available in the reverse direction - i.e. how does the server confirm client reception for a send/emit from the server? It would be nice to have a send/emit callback even just to indicate reception success. Didn't see this functionality documented anywhere...
Thanks!
Looking in the socket.io source I found that indeed ACKs are supported for server-sent messages (but not in broadcasts!) (lines 115-123 of socket.io/lib/socket.js):
if ('function' == typeof args[args.length - 1]) {
if (this._rooms || (this.flags && this.flags.broadcast)) {
throw new Error('Callbacks are not supported when broadcasting');
}
debug('emitting packet with ack id %d', this.nsp.ids);
this.acks[this.nsp.ids] = args.pop();
packet.id = this.nsp.ids++;
}
An example of how the ack should work (not tested):
// server-side:
io.on('msg', (data, ackCallback) => {
console.log('data from client', data);
ackCallback('roger roger');
});
// client-side:
socket.emit('msg', someData, (answer) => {
console.log('server\'s acknowledgement:', answer);
});
If we want to be 100% sure the reception success, just add ack call is not enough because we also need to know whether the ack call is run or not.
The socket.io 3.0 document add this timeout example to show how to do that. But the timeout value is the tricky one.
const withTimeout = (onSuccess, onTimeout, timeout) => {
let called = false;
const timer = setTimeout(() => {
if (called) return;
called = true;
onTimeout();
}, timeout);
return (...args) => {
if (called) return;
called = true;
clearTimeout(timer);
onSuccess.apply(this, args);
}
}
socket.emit("hello", 1, 2, withTimeout(() => {
console.log("success!");
}, () => {
console.log("timeout!");
}, 1000));
Yes we can send a response back to the server from the Client(as acknowledgment )
According to new documentation of socket(4.x)
Server-side
let dataToSend={
test:"fromServer"
}
socket.timeout(5000).emit("my-event",dataToSend, (err, response) => {
if (err) {
// the other side did not acknowledge the event in the given delay
} else {
console.log(response);
}
});
Cleint- side
socket.on("my-event", (data, callback) => {
// any logic for data(that data come from server i.e { test:"fromServer" }
callback({ test: "test" });
});
Related
My ampq system seems loosing messages, so I'd like a way to see if messages are effectively queued before being consumed.
I have several MicroServices communicating by amqp messages on NodeJs, using CloudAmqp. One of this microservice MS[B] generates .pdf, the process it's pretty heavy and requires about a minute for each request. So I send the .pdf asyncronously, triggering a webhook once finished, and generate once per time using a PreFetch = 1
So one MS[A] collects all the requests from the user, answers back to them saying "ok, request received, listen on the webhook" and in parallel it asks to the MS[B] to generate pdfs. MS[B] has prefetch=1, so consumes just one request per time. Once finished, sends the response to the callback queue of MS[A], which triggers the user webhook saying "the pdf, it's ready".
The problem is that MS[B] misses all the messages while busy:
it consumes one request from MS[A]
starts generating the .pdf
while generating, it discards all the other messages that MS[A] sends, as if there would be not any queue
it finishes the .pdf, sending ACK to MS[A]
then it starts again accepting messages, taking the last one received after being idle, losing all the previous ones.
Why? How can I find the problem, what could I monitor?
Communications between other MSs works well, with messages correctly ordered in queues. Just this one, with prefetch=1, loses messages.
I am NOT using the NO-ACK rule. I don't know what try, what test and what monitor to find the problem.
How can I see (if) messages are correctly queued before being consumed, ora just lost?
Below, the implementation of the messaging system
Channel Creation
/*
Starting Point of a connection to the CLOUDAMQP_URL server, then exec the callback
*/
start(callback) {
var self = this;
// if the connection is closed or fails to be established at all, we will reconnect
amqp.connect(process.env.CLOUDAMQP_URL + "?heartbeat=60")
.then(
function (conn) {
// create queues and consume mechanism
self.amqpConn = conn;
setTimeout(() => {
startPublisher();
}, 200);
setTimeout(() => {
createCallbackQueue();
}, 1000);
setTimeout(() => {
callback();
}, 2000);
});
// create publisher channels
function startPublisher() {
self.amqpConn.createConfirmChannel()
.then(function (ch) {
self.pubChannel = ch;
logger.debug("🗣️ pubChannel ready");
while (true) {
var m = self.offlinePubQueue.shift();
if (!m) break;
self.publish(m[0], // exchange
m[1], // routingKey
m[2], // content,
undefined // correlationId
);
}
});
}
// create callback channel
function createCallbackQueue() {
self.amqpConn.createChannel()
.then(function (channel) {
channel.assertQueue(self.CALLBACK_QUEUE_NAME, {
durable: true,
exclusive: true, // callback are exclusive
})
.then(function (q) {
logger.debug(" 👂 Waiting for RPC RESP in " + self.CALLBACK_QUEUE_NAME);
channel.consume(q.queue,
processCallback, {
noAck: false
}
);
});
// process messages of the callback
function processCallback(msg) {
var correlationId = msg.properties.correlationId;
}
//callback received
if (self.correlationIds_map[correlationId]) {
delete self.correlationIds_map[correlationId];
var content = JSON.parse(msg.content.toString());
self.eventEmitter.emit(correlationId, content);
}
}
});
}
return deferred.promise;
}
Consuming Messages
/*
#worker_queue - the name of the queue
*/
// Consume message from 'worker_queue', A worker that acks messages only if processed succesfully
startWorker(worker_queue, routes) {
var self = this;
logger.debug("startWorker " + self.CALLBACK_QUEUE_NAME);
var channel;
worker_queue = self.MICROSERVICE_NAME + worker_queue;
self.amqpConn.createChannel()
.then(
function (ch) {
channel = ch;
ch.prefetch(self.opt.prefetch); // = 1 for MS[B] generating pdf
channel.assertQueue(worker_queue, {
durable: true,
exclusive: true
})
.then(function (q) {
channel.consume(worker_queue, processMsg, {
noAck: false
});
});
});
// call the 'function from interface' passing params, and send the ACK
function processMsg(msg) {
work(msg)
.then(function (data) {
channel.ack(msg, false); // allUpTo = false
})
.catch(function (err) {
channel.ack(msg, false);
// channel.reject(msg, false); // requeue = false
// this.closeOnErr(e);
});
}
// execute the command, and queue back a response, checking if it's an error or not
function work(msg) {
var deferred = Q.defer();
var correlationId;
try {
correlationId = msg.properties.correlationId;
} catch (err) {}
work_function(msg.content, correlationId)
.then(function (resp) {
var content = {
data: resp
};
content = Buffer.from(JSON.stringify(content));
channel.sendToQueue(msg.properties.replyTo,
content, {
correlationId: correlationId,
content_type: 'application/json'
}
);
deferred.resolve(resp);
});
return deferred.promise;
}
}
Publish Messages
publish(exchange, routingKey, content, correlationId) {
var self = this;
var deferred = Q.defer();
self.correlationIds_map[correlationId] = true;
self.pubChannel.publish(exchange, routingKey, content,
{
replyTo: self.CALLBACK_QUEUE_NAME,
content_type : 'application/json',
correlationId: correlationId,
persistent : true
},
function(err, ok) {
if (err)
{
self.offlinePubQueue.push([exchange, routingKey, content]); // try again
self.pubChannel.connection.close();
deferred.resolve('requeued');
}
else
{
deferred.resolve(ok);
}
});
return deferred.promise;
}
This is on client side
socket.on('connect', () => {
console.log('client connect', socket.id);
const token = getToken();
socket.emit('token', token);
});
socket.on('message', data => {
....
//handle message
});
This is on server side
io.on('connection', (client) => {
client.on('token', token => {
verifyToken(token)
.then(({ _id: clientId }) => {
if (!clientId) return;
if (!connections[clientId]) {
connections[clientId] = new Map();
}
connections[clientId].set(client, 1);
client.on('disconnect', () => {
connections[clientId].delete(client);
});
});
});
});
}
async sendMessageToClients (workspaceId, message) {
const workspace = await getWorkspaceQuery(workspaceId);
if (!workspace) return;
const workspaceMembers = workspace.members.map(({ user }) => user);
for (const memberId of workspaceMembers) {
if (connections[memberId]) {
for (const clientConnection of connections[memberId].keys()) {
console.log('send to client', memberId, message.content, clientConnection.connected, clientConnection.id);
clientConnection.emit('message', message);
}
}
}
};
}
I purposely make a client offline by disconnect the wifi connection (make it in offline mode), what happen is that
a. if the disconnection is short, socket.id stay the same and I can get the buffered message send by other client when comes online;
b. but if I the disconnection is longer, the socket.id will change, and I can't get the buffered message send by other client when comes online.
How should I address that?
Since according to here the messages should be ideally buffered after reconnection.
I have a websocket in api gateway connected to a lambda that looks like this:
const AWS = require('aws-sdk');
const amqp = require('amqplib');
const api = new AWS.ApiGatewayManagementApi({
endpoint: 'MY_ENDPOINT',
});
async function sendMsgToApp(response, connectionId) {
console.log('=========== posting reply');
const params = {
ConnectionId: connectionId,
Data: Buffer.from(response),
};
return api.postToConnection(params).promise();
}
let rmqServerUrl =
'MY_RMQ_SERVER_URL';
let rmqServerConn = null;
exports.handler = async event => {
console.log('websocket event:', event);
const { routeKey: route, connectionId } = event.requestContext;
switch (route) {
case '$connect':
console.log('user connected');
const creds = event.queryStringParameters.x;
console.log('============ x.length:', creds.length);
const decodedCreds = Buffer.from(creds, 'base64').toString('utf-8');
try {
const conn = await amqp.connect(
`amqps://${decodedCreds}#${rmqServerUrl}`
);
const channel = await conn.createChannel();
console.log('============ created channel successfully:');
rmqServerConn = conn;
const [userId] = decodedCreds.split(':');
const { queue } = await channel.assertQueue(userId, {
durable: true,
autoDelete: false,
});
console.log('============ userId:', userId, 'queue:', queue);
channel.consume(queue, msg => {
console.log('========== msg:', msg);
const { content } = msg;
const msgString = content.toString('utf-8');
console.log('========== msgString:', msgString);
sendMsgToApp(msgString, connectionId)
.then(res => {
console.log(
'================= sent queued message to the app, will ack, outcome:',
res
);
try {
channel.ack(msg);
} catch (e) {
console.log(
'================= error acking message:',
e
);
}
})
.catch(e => {
console.log(
'================= error sending queued message to the app, will not ack, error:',
e
);
});
});
} catch (e) {
console.log(
'=========== error initializing amqp connection',
e
);
if (rmqServerConn) {
await rmqServerConn.close();
}
const response = {
statusCode: 401,
body: JSON.stringify('failed auth!'),
};
return response;
}
break;
case '$disconnect':
console.log('user disconnected');
if (rmqServerConn) {
await rmqServerConn.close();
}
break;
case 'message':
console.log('message route');
await sendMsgToApp('test', connectionId);
break;
default:
console.log('unknown route', route);
break;
}
const response = {
statusCode: 200,
body: JSON.stringify('Hello from websocket Lambda!'),
};
return response;
};
The amqp connection is for a rabbitmq server that's provisioned by amazonmq. The problem I have is that messages published to the queue either do not show up at all in the .consume callback, or they only show up after the websocket is disconnected and reconnected. Essentially they're missing until a point much later after which they show up unexpectedly. That's within the websocket. Even when they do show up, they don't get sent to the client (app in this case) that's connected to the websocket. What could be the problem here?
The problem here is that I had the wrong idea about how API Gateway's websockets work. API gateway maintains the websocket connection but not the lambda itself. I put my .consume subscription logic inside the lambda, which doesn't work because the lambda runs and terminates instead of being kept alive. A better method would be to make the queue an event source for the lambda. However this also didn't work for me because it requires you to know your queues when setting up the lambda. My queues are dynamically created so that violated the requirement. I ended up standing up a rmq server on a vps.
I have succesfully implemented this mechanism in my application:
https://vividcode.io/Spring-5-WebFlux-with-Server-Sent-Events/
I can receive events with curl every second, as shown in the example.
My problem is: I cannot receive these events in Angular 5. I have tried many things. Currently my service code looks like this:
public getMigrationProgress(processName: string): Observable<any> {
let headers: HttpHeaders = new HttpHeaders();
headers = headers.append('X-Authorization', this._sessionService.getAuthToken());
headers = headers.append('accept', 'text/event-stream');
let url = config.restApi.url + this.getResource() + '/' + processName;
return Observable.create(observer => {
let eventSource = new EventSourcePolyfill(url, { headers: headers });
eventSource.onmessage = (event => {
observer.next(event);
this.zone.run(() => {
console.log('prpprpr');
});
});
eventSource.onopen = (event) => {
observer.next(event);
};
eventSource.onerror = (error) => {
if (eventSource.readyState === 0) {
console.log('The stream has been closed by the server.');
eventSource.close();
observer.complete();
} else {
observer.error('EventSource error: ' + error);
}
};
});
}
It only opens connection, does not receive events (Method onopen works once, onmessage - never). Server sends them though.
Any ideas how to fix this?
Turned out that if you set event name on server, you cannot receive it by onmessage method.
In the example the event name was set to "random". In order to receive it you have to do it like this:
eventSource.addEventListener('random', function (event) {
console.log(event);
});
I am currently using this stack expres, socket.io, sessionstore. I followed the article here http://www.danielbaulig.de/socket-ioexpress/.
Well the problem is that i cannot modify the session values in socket.io callback.
Access from express side works well, the item get increased after each refresh.
app.get('/mysession', function(req, res) {
req.session.item++;
console.log(req.session);
res.render('session.jade', {
title: 'Sample title'
});
});
Using in socket.io side it does not and here is the problem, maybe i am setting the wrong object.
var io = io.listen(app);
io.sockets.on('connection', function(socket) {
var handshake = socket.handshake;
onlineCount++;
console.log('Well done id %s', handshake.sessionID);
handshake.session.item++;
console.log(handshake.session);
});
Here is bridge code.
io.set('authorization', function(data, accept) {
if (data.headers.cookie) {
data.cookie = parseCookie(data.headers.cookie);
data.sessionID = data.cookie['express.sid'];
sessionStore.get(data.sessionID, function(err, session) {
if (err || !session) {
accept('Error', false);
} else {
data.session = session;
accept(null, true);
}
});
} else {
return accept('No cookie tansmitted', false);
}
});
The only way I found to make this work is to grab the cookie from the request object on the connect event, parse it with your favourite cookie parser (I use connect.utils.parseCookie), and set it on that socket so that I may access it in future events:
socket.on('connection', function(client) {
var cookie = client.request.headers.cookie;
var pcookie = connect.utils.parseCookie(cookie);
var session_id = pcookie["connect.sid"];
if (session_id) {
sessionStore.get(session_id, function(err, sess) {
// do whatever you want with sess here
// ...
// if you want to "save" the session for future events
client.set('session_id', session_id);
}
}
});
The sessionStore API changed a little bit, now its sessionStore.load(sessionId, cb) instead of .get.