We're trying to send a message on an IBM MQ Message Queue.
We can contact the Queue Manager, but as soon as we call the MQMessage constructor, the process hangs. We suspect that this is actually because whatever AccessQueue is doing is failing.
using (MQQueueManager qMgr = new MQQueueManager(queueManagerName, connectionProperties))
{
// Set up the options on the queue we want to open
int openOptions = MQC.MQOO_OUTPUT + MQC.MQOO_FAIL_IF_QUIESCING;
// Now specify the queue that we want to open,and the open options
MQQueue system_default_local_queue =
qMgr.AccessQueue(sendQueueName, openOptions); // we suspect this is actually where things are wrong
// Define a WebSphere MQ message, writing some text in UTF format
MQMessage mqMessage = new MQMessage(); // ** THIS HANGS
mqMessage.WriteUTF(text);
// Specify the message options
MQPutMessageOptions pmo = new MQPutMessageOptions(); // accept the defaults,
// same as MQPMO_DEFAULT
// Put the message on the queue
system_default_local_queue.Put(mqMessage, pmo);
system_default_local_queue.Close();
// Disconnect from the queue manager
qMgr.Disconnect();
}
Is there an option we're not including or something we're missing?
We're using the the NuGet package WebSphereMqClient v8.0.0.7.
Update:
I'll include our connection properties for completeness based on feedback.
Hashtable connectionProperties = new Hashtable();
connectionProperties.Add(MQC.TRANSPORT_PROPERTY, MQC.TRANSPORT_MQSERIES_MANAGED);
connectionProperties.Add(MQC.USE_MQCSP_AUTHENTICATION_PROPERTY, true);
// Set up the rest of the connection properties, based on the
// connection type requested
// this is all for MQC.TRANSPORT_MQSERIES_MANAGED
connectionProperties.Add(MQC.HOST_NAME_PROPERTY, "myserver.com");
connectionProperties.Add(MQC.CHANNEL_PROPERTY, "MY.CHANNEL");
connectionProperties.Add(MQC.PORT_PROPERTY, 3223);
connectionProperties.Add(MQC.CONNECT_OPTIONS_PROPERTY, MQC.MQCNO_RECONNECT_Q_MGR);
In answer to a question from Roger, this is a Managed Queue.
You don't show us your connection properties, so I don't know if you set everything correctly. Did you make sure you set 'managed mode'? Also, why aren't you doing any print-outs of what the code has completed?
Here is a complete and working C# .NET/MQ program that uses 'managed mode' connection.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using IBM.WMQ;
/// <summary> Program Name
/// MQTest51
///
/// Description
/// This C# class will connect to a remote queue manager
/// and put a message to a queue under a managed .NET environment.
///
/// Sample Command Line Parameters
/// -h 127.0.0.1 -p 1414 -c TEST.CHL -m MQWT1 -q TEST.Q1
///
/// </summary>
namespace MQTest51
{
class MQTest51
{
private Hashtable inParms = null;
private Hashtable qMgrProp = null;
private System.String qManager;
private System.String outputQName;
/*
* The constructor
*/
public MQTest51()
: base()
{
}
/// <summary> Make sure the required parameters are present.</summary>
/// <returns> true/false
/// </returns>
private bool allParamsPresent()
{
bool b = inParms.ContainsKey("-h") && inParms.ContainsKey("-p") &&
inParms.ContainsKey("-c") && inParms.ContainsKey("-m") &&
inParms.ContainsKey("-q");
if (b)
{
try
{
System.Int32.Parse((System.String)inParms["-p"]);
}
catch (System.FormatException e)
{
b = false;
}
}
return b;
}
/// <summary> Extract the command-line parameters and initialize the MQ variables.</summary>
/// <param name="args">
/// </param>
/// <throws> IllegalArgumentException </throws>
private void init(System.String[] args)
{
inParms = Hashtable.Synchronized(new Hashtable());
if (args.Length > 0 && (args.Length % 2) == 0)
{
for (int i = 0; i < args.Length; i += 2)
{
inParms[args[i]] = args[i + 1];
}
}
else
{
throw new System.ArgumentException();
}
if (allParamsPresent())
{
qManager = ((System.String)inParms["-m"]);
outputQName = ((System.String)inParms["-q"]);
qMgrProp = new Hashtable();
qMgrProp.Add(MQC.TRANSPORT_PROPERTY, MQC.TRANSPORT_MQSERIES_MANAGED);
qMgrProp.Add(MQC.HOST_NAME_PROPERTY, ((System.String)inParms["-h"]));
qMgrProp.Add(MQC.CHANNEL_PROPERTY, ((System.String)inParms["-c"]));
try
{
qMgrProp.Add(MQC.PORT_PROPERTY, System.Int32.Parse((System.String)inParms["-p"]));
}
catch (System.FormatException e)
{
qMgrProp.Add(MQC.PORT_PROPERTY, 1414);
}
if (inParms.ContainsKey("-u"))
qMgrProp.Add(MQC.USER_ID_PROPERTY, ((System.String)inParms["-u"]));
if (inParms.ContainsKey("-x"))
qMgrProp.Add(MQC.PASSWORD_PROPERTY, ((System.String)inParms["-x"]));
if ( (inParms.ContainsKey("-u")) && (inParms.ContainsKey("-x")) )
qMgrProp.Add(MQC.USE_MQCSP_AUTHENTICATION_PROPERTY, true);
}
else
{
throw new System.ArgumentException();
}
}
/// <summary> Connect, open queue, write a message, close queue and disconnect.
///
/// </summary>
/// <throws> MQException </throws>
private void testSend()
{
MQQueueManager qMgr = null;
MQQueue outQ = null;
System.String line = "This is a test message embedded in the MQTest51 program.";
int openOptions = MQC.MQOO_OUTPUT + MQC.MQOO_FAIL_IF_QUIESCING;
MQPutMessageOptions pmo = new MQPutMessageOptions();
try
{
qMgr = new MQQueueManager(qManager, qMgrProp);
System.Console.Out.WriteLine("MQTest51 successfully connected to " + qManager);
outQ = qMgr.AccessQueue(outputQName, openOptions);
System.Console.Out.WriteLine("MQTest51 successfully opened " + outputQName);
// Define a simple MQ message, and write some text in UTF format..
MQMessage sendmsg = new MQMessage();
sendmsg.Format = MQC.MQFMT_STRING;
sendmsg.MessageType = MQC.MQMT_DATAGRAM;
sendmsg.MessageId = MQC.MQMI_NONE;
sendmsg.CorrelationId = MQC.MQCI_NONE;
sendmsg.WriteString(line);
// put the message on the outQ
outQ.Put(sendmsg, pmo);
System.Console.Out.WriteLine("Message Data>>>" + line);
}
catch (MQException mqex)
{
System.Console.Out.WriteLine("MQTest51 CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
catch (System.IO.IOException ioex)
{
System.Console.Out.WriteLine("MQTest51 ioex=" + ioex);
}
finally
{
try
{
if (outQ != null)
{
outQ.Close();
System.Console.Out.WriteLine("MQTest51 closed: " + outputQName);
}
}
catch (MQException mqex)
{
System.Console.Out.WriteLine("MQTest51 CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
try
{
if (qMgr != null)
{
qMgr.Disconnect();
System.Console.Out.WriteLine("MQTest51 disconnected from " + qManager);
}
}
catch (MQException mqex)
{
System.Console.Out.WriteLine("MQTest51 CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
}
}
/// <summary> main line</summary>
/// <param name="args">
/// </param>
// [STAThread]
public static void Main(System.String[] args)
{
MQTest51 write = new MQTest51();
try
{
write.init(args);
write.testSend();
}
catch (System.ArgumentException e)
{
System.Console.Out.WriteLine("Usage: MQTest51 -h host -p port -c channel -m QueueManagerName -q QueueName [-u userID] [-x passwd]");
System.Environment.Exit(1);
}
catch (MQException e)
{
System.Console.Out.WriteLine(e);
System.Environment.Exit(1);
}
System.Environment.Exit(0);
}
}
}
Related
I am working on IBM MQ testing. I able to inject my payloads request in MQ destination. I need to put the headers values in MQ to get the correct response. I used various option but still i am unable to put headers values in MQ server. Kindly let me know how can i solve this in JMeter.
Option 1 :
sendmsg = new MQMessage();
sendmsg.setStringProperty("QueryName", "GetPortfolio");
sendmsg.setStringProperty("Country", "LV");
Option 2:
rfh2.setFieldValue('usr', 'QueryName=', 'GetPortfolio')
Option 3:
SampleResult.setRequestHeaders("QueryName=GetPortfolio")
Option # 1 is the correct way of doing it.
Option # 2 could be used but why? And the parameter should not include the equals sign ( i.e. "=").
I have no idea what option # 3 is.
Here is a sample Java/MQ program that puts a message on a queue with named properties .
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Hashtable;
import com.ibm.mq.MQException;
import com.ibm.mq.MQMessage;
import com.ibm.mq.MQPutMessageOptions;
import com.ibm.mq.MQQueue;
import com.ibm.mq.MQQueueManager;
import com.ibm.mq.constants.CMQC;
/**
* Program Name
* MQTest11P
*
* Description
* This java class will connect to a remote queue manager with the
* MQ setting stored in a HashTable and put a message on a queue.
*
* Sample Command Line Parameters
* -m MQA1 -h 127.0.0.1 -p 1414 -c TEST.CHL -q TEST.Q1 -u UserID -x Password
*
* #author Roger Lacroix
*/
public class MQTest11P
{
private static final SimpleDateFormat LOGGER_TIMESTAMP = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");
private Hashtable<String,String> params;
private Hashtable<String,Object> mqht;
private String qMgrName;
private String outputQName;
/**
* The constructor
*/
public MQTest11P()
{
super();
params = new Hashtable<String,String>();
mqht = new Hashtable<String,Object>();
}
/**
* Make sure the required parameters are present.
* #return true/false
*/
private boolean allParamsPresent()
{
boolean b = params.containsKey("-h") && params.containsKey("-p") &&
params.containsKey("-c") && params.containsKey("-m") &&
params.containsKey("-q") &&
params.containsKey("-u") && params.containsKey("-x");
if (b)
{
try
{
Integer.parseInt((String) params.get("-p"));
}
catch (NumberFormatException e)
{
b = false;
}
}
return b;
}
/**
* Extract the command-line parameters and initialize the MQ HashTable.
* #param args
* #throws IllegalArgumentException
*/
private void init(String[] args) throws IllegalArgumentException
{
int port = 1414;
if (args.length > 0 && (args.length % 2) == 0)
{
for (int i = 0; i < args.length; i += 2)
{
params.put(args[i], args[i + 1]);
}
}
else
{
throw new IllegalArgumentException();
}
if (allParamsPresent())
{
qMgrName = (String) params.get("-m");
outputQName = (String) params.get("-q");
try
{
port = Integer.parseInt((String) params.get("-p"));
}
catch (NumberFormatException e)
{
port = 1414;
}
mqht.put(CMQC.CHANNEL_PROPERTY, params.get("-c"));
mqht.put(CMQC.HOST_NAME_PROPERTY, params.get("-h"));
mqht.put(CMQC.PORT_PROPERTY, new Integer(port));
mqht.put(CMQC.USER_ID_PROPERTY, params.get("-u"));
mqht.put(CMQC.PASSWORD_PROPERTY, params.get("-x"));
// I don't want to see MQ exceptions at the console.
MQException.log = null;
}
else
{
throw new IllegalArgumentException();
}
}
/**
* Connect, open queue, write a message, close queue and disconnect.
*
*/
private void testSend()
{
MQQueueManager qMgr = null;
MQQueue queue = null;
String msgData = "This is a test message from MQTest11P";
int openOptions = CMQC.MQOO_OUTPUT + CMQC.MQOO_FAIL_IF_QUIESCING;
MQPutMessageOptions pmo = new MQPutMessageOptions();
try
{
qMgr = new MQQueueManager(qMgrName, mqht);
MQTest11P.logger("successfully connected to "+ qMgrName);
queue = qMgr.accessQueue(outputQName, openOptions);
MQTest11P.logger("successfully opened "+ outputQName);
// Define a simple MQ message, and write some text
MQMessage sendmsg = new MQMessage();
sendmsg.format = CMQC.MQFMT_STRING;
sendmsg.feedback = CMQC.MQFB_NONE;
sendmsg.messageType = CMQC.MQMT_DATAGRAM;
sendmsg.messageId = CMQC.MQMI_NONE;
sendmsg.correlationId = CMQC.MQCI_NONE;
// Write message data
sendmsg.writeString(msgData);
/**
* Set named properties aka message properties
* that will create a JMS message.
*/
// sendmsg.setStringProperty("mcd.Msd", "jms_text");
// sendmsg.setStringProperty("jms.Dst", "queue:///"+outputQName);
// sendmsg.setStringProperty("jms.Pri", "0");
sendmsg.setStringProperty("QueryName", "GetPortfolio");
sendmsg.setStringProperty("Country", "LV");
// put the message on the queue
queue.put(sendmsg, pmo);
MQTest11P.logger("Message Data>>>" + msgData);
}
catch (MQException e)
{
MQTest11P.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
}
catch (IOException e)
{
MQTest11P.logger("IOException:" +e.getLocalizedMessage());
}
finally
{
try
{
if (queue != null)
{
queue.close();
MQTest11P.logger("closed: "+ outputQName);
}
}
catch (MQException e)
{
MQTest11P.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
}
try
{
if (qMgr != null)
{
qMgr.disconnect();
MQTest11P.logger("disconnected from "+ qMgrName);
}
}
catch (MQException e)
{
MQTest11P.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
}
}
}
/**
* A simple logger method
* #param data
*/
public static void logger(String data)
{
String className = Thread.currentThread().getStackTrace()[2].getClassName();
// Remove the package info.
if ( (className != null) && (className.lastIndexOf('.') != -1) )
className = className.substring(className.lastIndexOf('.')+1);
System.out.println(LOGGER_TIMESTAMP.format(new Date())+" "+className+": "+Thread.currentThread().getStackTrace()[2].getMethodName()+": "+data);
}
/**
* main line
* #param args
*/
public static void main(String[] args)
{
MQTest11P write = new MQTest11P();
try
{
write.init(args);
write.testSend();
}
catch (IllegalArgumentException e)
{
MQTest11P.logger("Usage: java MQTest11P -m QueueManagerName -h host -p port -c channel -q QueueName -u UserID -x Password");
System.exit(1);
}
System.exit(0);
}
}
When I look at the message in the queue with MQ Visual Edit, I see:
So, your 2 named properties are with the message.
The question becomes do you want JUST named properties attached to the message or are you really looking to create a JMS message with named properties?
Note: A JMS message is internally defined as an MQRFH2 message. i.e. JMS message ==> MQRFH2 message but the opposite may not be true.
Hence, if you want a JMS message with named properties then you need to uncomment those 3 lines in MQTest11P.java above.
i.e.
sendmsg.setStringProperty("mcd.Msd", "jms_text");
sendmsg.setStringProperty("jms.Dst", "queue:///"+outputQName);
sendmsg.setStringProperty("jms.Pri", "0");
This is not about Delayed message delivery. Lately I've noticed my endpoint has a delay in handling a message when looking at when the message is taken from the RabbitMQ queue.
I was wondering if there is some configuration option I’m overlooking that has an impact on the delay between the receipt of the message from RabbitMQ by NServiceBus and the actual handling of the message (command) in the IHandleMessages<> handler.
I’m seeing a consistent delay when comparing the NServiceBus logfile and the logfile of the handler. It does vary a bit between client and server but in a given session it’s always constant. Now I’m running through all the options that I can find but there’s no real pointer for me out in the wild.
The thing is I’m running a workflow solution that uses NServiceBus to do all the messaging between the workflow server (based on MS WF) and 50/60 clients. The server runs on NServiceBus 7.6.0 but all client still use NServiceBus 5.2.26. I decided to write a small test client that also runs on NServiceBus 7.6.0 but that also shows the delay albeit a smaller one. But the delay between individual commands is constant at 6.5 seconds in the test client. So the message is retrieved by NSB from the queue and 6.5 seconds the IHandleMessages<> handler is invoked.
I’ve tried a couple of things (direct routing vs conventional routing, making sure license files where distributed properly for example). The only thing I’m able to come up right now is flat out making a new test server of the workflow server I’m using that is only able to accept 1 command, generating a reply and nothing more and see if that makes a difference.
Configuration wise the solution runs on a Windows Server 2016 having 4 cores, RabbitMQ 3.7.21 and NServiceBus 7.6.0. Clients run on Windows 10 machines using NServiceBus 5.2.26 but the test client uses NServiceBus 7.6.0.
Now for the logs. This is the NServicebus log when it receives the GetVersionCommand:
2022-01-20 16:09:58.104 DEBUG Processing message type: HyFlo.Service.Messages.Commands.Info.GetVersionCommand
Message headers:
CorrelationId : 619d678d-25bf-46c0-b5c0-57c624e3d557
NServiceBus.MessageId : 548813d5-2465-4ff5-904d-ae2300f9ee23
NServiceBus.MessageIntent : Send
NServiceBus.ConversationId : 367575b0-7bee-40d0-90ca-ae2300f9ee24
NServiceBus.CorrelationId : 548813d5-2465-4ff5-904d-ae2300f9ee23
NServiceBus.TimeToBeReceived : 00:10:00
NServiceBus.ReplyToAddress : Hyflo.Client.Test
NServiceBus.OriginatingMachine : DYP297
NServiceBus.OriginatingEndpoint : Hyflo.Client.Test
$.diagnostics.originating.hostid : 3a3bbd6661214b655aa6dbcd95d112a2
NServiceBus.ContentType : text/xml
NServiceBus.EnclosedMessageTypes : HyFlo.Service.Messages.Commands.Info.GetVersionCommand, HyFlo.Service.Messages, Version=4.10.3.26734, Culture=neutral, PublicKeyToken=null;HyFlo.Types.Interfaces.IInfoCommand, HyFlo.Types, Version=4.10.8.20782, Culture=neutral, PublicKeyToken=null
NServiceBus.Version : 7.6.0
NServiceBus.TimeSent : 2022-01-20 15:09:58:107461 Z
NServiceBus.NonDurableMessage : False
Handlers to invoke:
HyFlo.Service.CommandHandlers.Info.GetVersionHandler
Then the custom log of the GetVersionHandler:
2022-01-20 16:10:04.6516 TRACE = New Message =====================================================================================================================================
2022-01-20 16:10:04.6516 TRACE CommandEventId:9125b036-c315-484e-9948-889ed1e56587: New message handled by handler of type 'HyFlo.Service.CommandHandlers.Info.GetVersionHandler' with CorrelationId '619d678d-25bf-46c0-b5c0-57c624e3d557' ..
2022-01-20 16:10:04.6516 TRACE ===================================================================================================================================================
2022-01-20 16:10:04.6516 INFO Cache contains a systemstatus ..
2022-01-20 16:10:04.6516 INFO Retrieved systemstatus from cache is: 0
2022-01-20 16:10:04.6516 INFO Running the injected code ..
2022-01-20 16:10:04.6672 TRACE ===================================================================================================================================================
2022-01-20 16:10:04.6672 TRACE CommandEventId:9125b036-c315-484e-9948-889ed1e56587: Message processed by handler of type 'HyFlo.Service.CommandHandlers.Info.GetVersionHandler' with CorrelationId '619d678d-25bf-46c0-b5c0-57c624e3d557' ..
2022-01-20 16:10:04.6672 TRACE Elapsed time: 00:00:00.0156159 ..
2022-01-20 16:10:04.6672 TRACE = End of Message ==================================================================================================================================
The command is received at 2022-01-20 16:09:58.104 but is delivered at the Handler at 2022-01-20 16:10:04.6516 and this delay is quite consistent between this and other commands. I just can't get my head around why this delay is there.
The endpoint configuration is the following:
internal class ProgramService : ServiceBase
{
private IEndpointInstance _endpointInstance = null;
/// <summary>
/// Determines if application is ran as service or just as console application
/// </summary>
/// <param name="name"></param>
/// <returns>true if it's a service</returns>
private static bool IsService(string name)
{
if (!Environment.UserInteractive) return true;
var sc = new ServiceController(name);
try
{
return sc.Status == ServiceControllerStatus.StartPending;
}
catch (InvalidOperationException)
{
return false;
}
}
/// <summary>
/// Main entry
/// </summary>
private static void Main()
{
using (var service = new ProgramService())
{
// so we can run interactive from Visual Studio or as a windows service
if (!IsService("HyFlo.Service"))
{
Console.CancelKeyPress += (sender, e) => { service.OnStop(); };
service.OnStart(null);
Console.WriteLine("\r\nPress enter key to stop program\r\n");
Console.Read();
service.OnStop();
return;
}
Run(service);
}
}
/// <summary>
/// On critical errors bail out
/// </summary>
/// <param name="errorMessage"></param>
/// <param name="exception"></param>
async Task OnCriticalError(ICriticalErrorContext context)
{
var fatalMessage = $"The following critical error was encountered:\n{context.Error}\nProcess is shutting down.";
try
{
Console.WriteLine("fatalMessage: " + fatalMessage);
TraceWriter.Error(fatalMessage);
if (context.Exception.InnerException != null)
{
TraceWriter.Error("innerException message: " + context.Exception.InnerException.Message + #"\n" +
context.Exception.InnerException.StackTrace);
}
await context.Stop().ConfigureAwait(false);
}
finally
{
Environment.FailFast(fatalMessage, context.Exception);
}
}
/// <summary>
/// Starting the service
/// </summary>
/// <param name="args"></param>
private async Task AsyncOnStart()
{
TraceWriter.Trace("AsyncOnStart() running .. ");
try
{
TraceWriter.Info("Running configuration management ..");
var config =
ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None);
var section =
config.GetSection("connectionStrings");
if (!section.SectionInformation.IsProtected)
{
section.SectionInformation.ProtectSection("DPAPIProtection");
section.SectionInformation.ForceSave = true;
config.Save(ConfigurationSaveMode.Modified);
}
var endpointConfig = new EndpointConfiguration("hyflo.service");
endpointConfig.EnableInstallers();
endpointConfig.SendFailedMessagesTo("hyflo.service.errors");
endpointConfig.AuditProcessedMessagesTo("hyflo.service.audit");
endpointConfig.UseSerialization<XmlSerializer>();
endpointConfig.DefineCriticalErrorAction(OnCriticalError);
endpointConfig.LimitMessageProcessingConcurrencyTo(8);
var persistenceMethod = ConfigurationManager.AppSettings["NServicebusPersistence"];
if (persistenceMethod.ToUpper() == "RAVENDB")
{
string connectionString = ConfigurationManager.ConnectionStrings["NServiceBus/Persistence"].ConnectionString;
TraceWriter.Info($"Setting persistence to RavenDBPersistence based on connectionString '{connectionString}' .. ");
var documentStore = new Raven.Client.Document.DocumentStore
{
ConnectionStringName = "NServiceBus/Persistence",
DefaultDatabase = "HyFlo.Service"
};
documentStore.Initialize();
endpointConfig.UsePersistence<RavenDBPersistence>().SetDefaultDocumentStore(documentStore);
}
else
{
TraceWriter.Info("Setting persistence to InMemoryPersistence .. ");
endpointConfig.UsePersistence<InMemoryPersistence>();
}
var transportConnection = ConfigurationManager.ConnectionStrings[#"NServiceBus/Transport"];
string transportConnectionString = transportConnection.ConnectionString;
if (String.IsNullOrEmpty(transportConnectionString))
{
transportConnectionString = "host=localhost";
}
TraceWriter.Info($"Configuring RabbitMQTransport for connection '{transportConnectionString}' .. ");
var transport = endpointConfig.UseTransport<RabbitMQTransport>();
transport.ConnectionString(transportConnectionString);
transport.UseConventionalRoutingTopology();
string databusBasePath = ConfigurationManager.AppSettings["DataBusBasePath"] ?? "";
TraceWriter.Info($"Setting Databus's basepath to '{databusBasePath}' ..");
endpointConfig.UseDataBus<FileShareDataBus>().BasePath(databusBasePath);
TraceWriter.Info("Scannning for Hyflo assemblies .. ");
List<string> hyfloAssemblies =
Directory.GetFiles(Directory.GetCurrentDirectory(), "HyFlo.*dll", SearchOption.TopDirectoryOnly).ToList();
TraceWriter.Info("Initializing Autofac with assemblies .. ");
foreach (string assemblyName in hyfloAssemblies)
{
TraceWriter.Info($"Scanning '{assemblyName}' for an Autofac module .. ");
}
try
{
var containerSettings = endpointConfig.UseContainer(new AutofacServiceProviderFactory());
containerSettings.ConfigureContainer(containerBuilder =>
{
var loadedAssemblies = hyfloAssemblies.Select(Assembly.LoadFile).ToList();
containerBuilder.RegisterAssemblyModules(loadedAssemblies.ToArray());
});
}
catch (Exception ex)
{
TraceWriter.Error($"{ex.Message}\n{ex.StackTrace}\n{ex.InnerException?.Source}");
await OnCriticalError(new CriticalErrorContext(null, $"Exception occurred during initialization. Exception is: {ex.Message}\n{ex.StackTrace}\n{ex.InnerException?.Source}", ex));
}
TraceWriter.Info("Setting up default message conventions ..");
var conventions = endpointConfig.Conventions();
conventions.DefiningTimeToBeReceivedAs(type => TimeSpan.FromMinutes(10));
var logFactory = LogManager.Use<DefaultFactory>();
logFactory.Level(LogLevel.Debug);
endpointConfig.EnableFeature<HyfloInitializationFeature>();
TraceWriter.Info("Initialized! Now starting Hyflo ..");
_endpointInstance = await Endpoint.Start(endpointConfig);
TraceWriter.Info("Hyflo started ..");
}
catch (Exception exception)
{
TraceWriter.Error($"{exception.Message}\n{exception.StackTrace}\n{exception.InnerException?.Source}");
await OnCriticalError(new CriticalErrorContext(null, "Failed to start the bus.", exception));
}
}
/// <summary>
/// Stopping the service
/// </summary>
/// <returns></returns>
private Task AsyncOnStop()
{
TraceWriter.Info("Shutting down Hyflo ..");
if (_endpointInstance != null)
{
return _endpointInstance.Stop();
}
return Task.CompletedTask;
}
// <summary>
/// Starting the service
/// </summary>
/// <param name="args"></param>
protected override void OnStart(string[] args)
{
TraceWriter.Trace("ProgramService.OnStart() ..");
AsyncOnStart().GetAwaiter().GetResult();
}
/// <summary>
/// Stopping the service
/// </summary>
protected override void OnStop()
{
TraceWriter.Trace("ProgramService.OnStop() ..");
AsyncOnStop().GetAwaiter().GetResult();
}
}
The HyfloInitializationFeature is a startup task that sends a list of all current workflows to the connected clients.
using HyFlo.Tasks.Interfaces;
using NServiceBus.Features;
namespace HyFlo.Service
{
public class HyfloInitializationFeature : Feature
{
protected override void Setup(FeatureConfigurationContext context)
{
context.RegisterStartupTask(c => new SystemStartupTask(c.Build<IGetInfoOnAllWorkflowsTask>(), c.Build<ISystemTestTask>()));
}
}
}
This is the SystemStartupTask:
namespace HyFlo.Service
{
public class SystemStartupTask : FeatureStartupTask, IDisposable
{
private readonly IGetInfoOnAllWorkflowsTask _getInfoOnAllWorkflowsTask;
private readonly ISystemTestTask _systemTestTask;
public SystemStartupTask(IGetInfoOnAllWorkflowsTask getInfoOnAllWorkflowsTask, ISystemTestTask systemTestTask)
{
_getInfoOnAllWorkflowsTask = getInfoOnAllWorkflowsTask;
_systemTestTask = systemTestTask;
}
public void Dispose()
{
}
/// <summary>
/// OnStart
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
protected override async Task OnStart(IMessageSession session)
{
TraceWriter.Trace($"==> Running SystemStartupTask.OnStart() ..");
DeleteHeartbeatIndicator();
ObjectCache cache = MemoryCache.Default;
var policy = new CacheItemPolicy { SlidingExpiration = (TimeSpan.FromSeconds(15)) };
bool systemOk = false;
if (_systemTestTask != null)
{
_systemTestTask.ContextWrapper = new MessageContextWrapper { Session = session };
_systemTestTask.QuickTest = false;
_systemTestTask.TestComponentsMask = 31;
_systemTestTask.SystemStartup = true;
if (_systemTestTask.Run())
{
var systemResultTexts = new List<string>();
if (int.TryParse(_systemTestTask.Results.ToString(), out int systemsResult))
{
if ((systemsResult & 1) == 1)
{
systemResultTexts.Add("HSB offline");
}
if ((systemsResult & 2) == 2)
{
systemResultTexts.Add("HDM offline");
}
if ((systemsResult & 4) == 4)
{
systemResultTexts.Add("SqlServer offline");
}
if ((systemsResult & 8) == 8)
{
systemResultTexts.Add("Workflow Queueing offline");
}
}
TraceWriter.Trace(
$"SystemStartupTask: Results returned by systemtest: '{(!systemResultTexts.Any() ? "All online" : String.Join(",", systemResultTexts))}' ..");
if (!_systemTestTask.Failure && !systemResultTexts.Any())
{
TraceWriter.Info("HyFlo dependencies all up and running ..");
systemOk = true;
// for caching purposes
if (_getInfoOnAllWorkflowsTask != null)
{
_getInfoOnAllWorkflowsTask.UserId = "";
_getInfoOnAllWorkflowsTask.Run();
}
}
else
{
TraceWriter.Warn("HyFlo can't be started. One or more of its dependencies returned a failure!");
}
}
}
else
{
TraceWriter.Warn("A general failure occurred during the Hyflo systemcheck!");
}
var allworkflowStates = new WorkflowState[0];
if (_getInfoOnAllWorkflowsTask != null)
{
if (!_getInfoOnAllWorkflowsTask.Failure && _getInfoOnAllWorkflowsTask.Results is List<WorkflowState>)
{
allworkflowStates = (_getInfoOnAllWorkflowsTask.Results as List<WorkflowState>).ToArray();
}
TraceWriter.Info(
$"Workflowstate retrieval completed. Failure: '{_getInfoOnAllWorkflowsTask.Failure}', Results: '{allworkflowStates.Count()}' workflow{(allworkflowStates.Count() == 1 ? "" : "s")} found ..");
var timeStamp = DateTime.Now;
TraceWriter.Trace("Sending IStartedHyflo # GUI with the following parameters:\n" +
$"\tErrorCode: '{Convert.ToInt32(_systemTestTask?.Results)}'\n" +
$"\tFailure: '{!systemOk}'\n" + $"\tTimeStamp: '{timeStamp}'\n" +
$"\tResulting workflowcount: '{allworkflowStates.Length}'\n");
await session.Publish<IStartedHyflo>(evt =>
{
evt.ErrorCode = Convert.ToInt32(_systemTestTask?.Results);
evt.Failure = !systemOk;
evt.TimeStamp = timeStamp;
evt.Result = new DataBusProperty<WorkflowState[]>(allworkflowStates);
}).ConfigureAwait(false);
cache.Set("SystemTest", systemOk, policy);
}
}
/// <summary>
/// OnStop
/// </summary>
/// <param name="session"></param>
/// <returns></returns>
protected override async Task OnStop(IMessageSession session)
{
TraceWriter.Trace($"==> Running SystemStartupTask.OnStop() ..");
TraceWriter.Info("Deleting heartbeat indicator .. ");
DeleteHeartbeatIndicator();
var timeStamp = DateTime.Now;
TraceWriter.Trace("Sending IShutdownHyflo # GUI with the following parameters:\n" + "\tErrorCode: '0'\n" +
"\tFailure: 'false'\n" + $"\tTimeStamp: '{timeStamp}'");
await session.Publish<IShutdownHyflo>(evt =>
{
evt.ErrorCode = 0;
evt.Failure = false;
evt.TimeStamp = timeStamp;
}).ConfigureAwait(false);
TraceWriter.Info("HyFlo has shutted down .. ");
}
/// <summary>
/// Function deletes the heartbeat.txt file that indicates wether a heartbeat flow is running or not
/// </summary>
private void DeleteHeartbeatIndicator()
{
string stateFolder = ConfigurationManager.AppSettings["StateFolder"];
string fullStateFolder = stateFolder.IndexOf(":", StringComparison.Ordinal) == -1
? $"{AppDomain.CurrentDomain.BaseDirectory}{stateFolder}"
: stateFolder;
string fileName = $#"{fullStateFolder}\heartbeat.txt";
if (File.Exists(fileName))
{
File.Delete(fileName);
}
}
}
}
The GetVersionHandler is the following:
namespace HyFlo.Service.CommandHandlers.Info
{
public class GetVersionHandler : HandlerBase, IHandleMessages<GetVersionCommand>
{
public GetVersionHandler(ISystemTestTask systemTestTask, ITaskBuilderController taskBuilderController)
{
_systemTestTask = systemTestTask;
_taskBuilderController = taskBuilderController;
}
public virtual async Task Handle(GetVersionCommand message, IMessageHandlerContext context)
{
if (message == null)
return;
await RunDirect(async () =>
{
var assemblyVersion = Assembly.GetAssembly(typeof(GetVersionHandler)).GetName().Version;
var reply = new GetVersionReply
{
Failure = false,
Version = $"{assemblyVersion.Major}.{assemblyVersion.Minor}",
TimeStamp = DateTime.Now,
TaskId = CommandEventId,
ErrorCode = 0
};
await Retry.ExecuteAsync(async () =>
{
var replyOptions = new ReplyOptions();
replyOptions.SetHeader("CorrelationId", CorrelationId);
await context.Reply(reply, replyOptions);
});
}, new MessageContextWrapper { HandlerContext = context });
}
}
}
The RunDirect method is located in the HandlerBase class and one of the first things it does is log that a new message is being handled.
public async Task RunDirect(Action codeToRun, IMessageContextWrapper contextWrapper)
{
_taskBuilderController.HandlerBase = this;
CommandEventId = Guid.NewGuid();
ContextWrapper = contextWrapper;
CorrelationId = GetHeaderValue("CorrelationId");
string fullMethodName = GetType().ToString();
TraceWriter.Trace("= New Message =====================================================================================================================================");
TraceWriter.Trace($"CommandEventId:{CommandEventId}: New message handled by handler of type '{fullMethodName}' with CorrelationId '{CorrelationId}' .. ");
TraceWriter.Trace("===================================================================================================================================================");
...
The last thing is the test client I've used to send this GetVersionCommand:
namespace HPS_EndpointTest
{
internal class Program
{
static void Main(string[] args)
{
var endpointConfiguration = new EndpointConfiguration("Hyflo.Client.Test");
endpointConfiguration.EnableInstallers();
endpointConfiguration.UsePersistence<InMemoryPersistence>();
endpointConfiguration.PurgeOnStartup(true);
var transport = endpointConfiguration.UseTransport<RabbitMQTransport>();
endpointConfiguration.UseTransport<RabbitMQTransport>().ConnectionString("host=hswv0601;username=hyflo;password=hyflo");
endpointConfiguration.UseTransport<RabbitMQTransport>().UseDirectRoutingTopology();
IEndpointInstance endpointInstance = null;
AsyncPump.Run(async delegate
{
endpointInstance = await Endpoint.Start(endpointConfiguration).ConfigureAwait(false);
});
Console.WriteLine(DateTime.Now.ToString("dd-MM-yyyyTHH:mm:ss.ffff") + ": Client started ..");
var correlationId = Guid.NewGuid();
string destination = "hyflo.service";
var sendOptions = new SendOptions();
sendOptions.SetHeader("CorrelationId", correlationId.ToString());
sendOptions.SetDestination(destination);
sendOptions.RequireImmediateDispatch();
var versionMessage = new HyFlo.Service.Messages.Commands.Info.GetVersionCommand();
Console.WriteLine(DateTime.Now.ToString("dd-MM-yyyyTHH:mm:ss.ffff") + ": Sending message ..");
AsyncPump.Run(async delegate
{
if (endpointInstance != null)
{
await endpointInstance.Send(versionMessage, sendOptions).ConfigureAwait(false);
}
});
Console.WriteLine(DateTime.Now.ToString("dd-MM-yyyyTHH:mm:ss.ffff") + ": Message sent!");
while (true)
{
var key = Console.ReadKey();
if (key.Key == ConsoleKey.Enter)
{
break;
}
}
AsyncPump.Run(async delegate
{
await endpointInstance.Stop().ConfigureAwait(false);
});
}
}
public class RetrieveAllWorkflowsHandler : IHandleMessages<HyFlo.Service.Messages.Events.IRetrievedAllWorkflowsEvent>
{
public Task Handle(HyFlo.Service.Messages.Events.IRetrievedAllWorkflowsEvent message, IMessageHandlerContext context)
{
Console.WriteLine(DateTime.Now.ToString("dd-MM-yyyyTHH:mm:ss.ffff") + ": IRetrievedAllWorkflowEvents reply received ..");
return Task.CompletedTask;
}
}
public class RetrieveVersionHandler : IHandleMessages<GetVersionReply>
{
public Task Handle(GetVersionReply message, IMessageHandlerContext context)
{
Console.WriteLine(DateTime.Now.ToString("dd-MM-yyyyTHH:mm:ss.ffff") + ": GetVersionReply reply received ..");
return Task.CompletedTask;
}
}
}
I just don't understand why this delay is there.
What I'm doing right now is create a completely new endpoint with limited functionality (ie only supporting the GetVersionCommand) and see what happens with that. If the delay isn't there I'll expand support with other commands and see what happens then. Hope any of you guys sees this and slaps him/herself to the head and replying "but you forgot so and so".
Thanks in advance!!
With help of Daniel Marbach I managed to pin point the problem. Because of a complex object graph in one of the classes of the solution (the class that is the primary access point to the solution in fact) the delay occurred. Now when creating either specialised classes or injecting the actual needed dependency there was virtually no delay.
So the final solution was to alter each and every IHandleMessages-handler to have it injected a targeted dependency with which the handler could work.
Hi I'm new on IBM MQ environment so I need your help....
I need to develope a script that check an IBMMQ (outside my factory) queue to verify if there are messages waiting to be get from my application-server (I mean if my apllication-server lost the connection to IBMMQ for any reason, I want to be informed, in this mode I can take the correction).
If is possible I need the script in PowereShell or C#.
Thanks a lot.
There is sample code that can get you started - https://github.com/ibm-messaging/mq-dev-patterns/blob/master/dotnet/dotNetGet.cs
replace the CreateConsumer method invocation with CreateBrowser
ie. the modified snippet becomes
private void ReceiveMessagesFromEndpoint(IConnectionFactory cf)
{
IConnection connectionWMQ;
ISession sessionWMQ;
IDestination destination;
IMessageBrowser browser;
ITextMessage textMessage;
// Create connection.
connectionWMQ = cf.CreateConnection();
Console.WriteLine("Connection created");
// Create session
sessionWMQ = connectionWMQ.CreateSession(false, AcknowledgeMode.AutoAcknowledge);
Console.WriteLine("Session created");
// Create destination
destination = sessionWMQ.CreateQueue(env.Conn.queue_name);
Console.WriteLine("Destination created");
// Create browser
browser = sessionWMQ.CreateBrowser(destination);
Console.WriteLine("Browser created");
...
Instructions on setup are available in the associated readme - https://github.com/ibm-messaging/mq-dev-patterns/blob/master/dotnet/README.md
Do not browse the queue to get the current queue depth as it will cause an excessive amount of network traffic and is totally unnecessary.
Here is a C#/.NET/MQ sample program (fully functional) that will inquire on the queue for its current depth.
using System;
using System.Collections;
using System.Collections.Generic;
using System.Configuration;
using System.Text;
using IBM.WMQ;
/// <summary> Program Name
/// MQTest75
///
/// Description
/// This C# class will connect to a remote queue manager
/// and inquire on the current depth of the queue using a managed .NET environment.
///
/// </summary>
/// <author> Roger Lacroix
/// </author>
namespace MQTest75
{
class MQTest75
{
private Hashtable qMgrProp = null;
private System.String qManager;
private System.String inputQName;
/*
* The constructor
*/
public MQTest75()
: base()
{
}
/// <summary> Make sure the required parameters are present.</summary>
/// <returns> true/false
/// </returns>
private bool allParamsPresent()
{
bool b = false;
if ( (ConfigurationManager.AppSettings["ConnectionName"] != null) &&
(ConfigurationManager.AppSettings["Port"] != null) &&
(ConfigurationManager.AppSettings["ChannelName"] != null) &&
(ConfigurationManager.AppSettings["QMgrName"] != null) &&
(ConfigurationManager.AppSettings["QueueName"] != null) )
{
try
{
System.Int32.Parse(ConfigurationManager.AppSettings["Port"]);
b = true;
}
catch (System.FormatException e)
{
b = false;
}
}
return b;
}
/// <summary> Extract the configuration applicaiton settings and initialize the MQ variables.</summary>
/// <param name="args">
/// </param>
/// <throws> IllegalArgumentException </throws>
private void init(System.String[] args)
{
if (allParamsPresent())
{
qManager = ConfigurationManager.AppSettings["QMgrName"];
inputQName = ConfigurationManager.AppSettings["QueueName"];
qMgrProp = new Hashtable();
qMgrProp.Add(MQC.TRANSPORT_PROPERTY, MQC.TRANSPORT_MQSERIES_MANAGED);
qMgrProp.Add(MQC.HOST_NAME_PROPERTY, ConfigurationManager.AppSettings["ConnectionName"]);
qMgrProp.Add(MQC.CHANNEL_PROPERTY, ConfigurationManager.AppSettings["ChannelName"]);
try
{
qMgrProp.Add(MQC.PORT_PROPERTY, System.Int32.Parse(ConfigurationManager.AppSettings["Port"]));
}
catch (System.FormatException e)
{
qMgrProp.Add(MQC.PORT_PROPERTY, 1414);
}
if (ConfigurationManager.AppSettings["UserId"] != null)
qMgrProp.Add(MQC.USER_ID_PROPERTY, ConfigurationManager.AppSettings["UserId"]);
if (ConfigurationManager.AppSettings["Password"] != null)
qMgrProp.Add(MQC.PASSWORD_PROPERTY, ConfigurationManager.AppSettings["Password"]);
logger("Parameters:");
logger(" QMgrName ='" + qManager + "'");
logger(" Queue Name ='" + inputQName + "'");
logger("Connection values:");
foreach (DictionaryEntry de in qMgrProp)
{
logger(" " + de.Key + " = '" + de.Value + "'");
}
}
else
{
throw new System.ArgumentException();
}
}
/// <summary> Connect, open queue, output the current queue depth, close queue and disconnect.</summary>
/// <throws> MQException </throws>
private void handleIt()
{
MQQueueManager qMgr = null;
MQQueue inQ = null;
int openOptions = MQC.MQOO_INQUIRE + MQC.MQOO_FAIL_IF_QUIESCING;
try
{
qMgr = new MQQueueManager(qManager, qMgrProp);
logger("Successfully connected to " + qManager);
inQ = qMgr.AccessQueue(inputQName, openOptions);
logger("Successfully opened " + inputQName);
logger("Current queue depth is " + inQ.CurrentDepth);
}
catch (MQException mqex)
{
logger("CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
catch (System.IO.IOException ioex)
{
logger("Error: ioex=" + ioex);
}
finally
{
try
{
if (inQ != null)
{
inQ.Close();
logger("Closed: " + inputQName);
}
}
catch (MQException mqex)
{
logger("CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
try
{
if (qMgr != null)
{
qMgr.Disconnect();
logger("Disconnected from " + qManager);
}
}
catch (MQException mqex)
{
logger("CC=" + mqex.CompletionCode + " : RC=" + mqex.ReasonCode);
}
}
}
/// <summary> Output the log message to stdio.</summary>
/// <param name="data">
/// </param>
private void logger(String data)
{
DateTime myDateTime = DateTime.Now;
System.Console.Out.WriteLine(myDateTime.ToString("yyyy/MM/dd HH:mm:ss.fff") + " " + this.GetType().Name + ": " + data);
}
/// <summary> main line</summary>
/// <param name="args">
/// </param>
// [STAThread]
public static void Main(System.String[] args)
{
MQTest75 mqt = new MQTest75();
try
{
mqt.init(args);
mqt.handleIt();
}
catch (System.ArgumentException e)
{
System.Console.Out.WriteLine("Check your configuration file for an incorrect parameter.");
System.Environment.Exit(1);
}
catch (MQException e)
{
System.Console.Out.WriteLine(e);
System.Environment.Exit(1);
}
System.Environment.Exit(0);
}
}
}
Just create a configuration called MQTest75.exe.config and put the following in it:
<configuration>
<appSettings>
<add key="QMgrName" value="MQA1"/>
<add key="ConnectionName" value="10.10.10.10"/>
<add key="Port" value="1414"/>
<add key="ChannelName" value="TEST.CHL"/>
<add key="QueueName" value="TEST.Q1"/>
<add key="UserId" value="tester"/>
<add key="Password" value="mypwd"/>
</appSettings>
</configuration>
I'm wondering if there is a tool or lib that can move messages between queues?
Currently, i'm doing something like below
public static void ProcessQueueMessage([QueueTrigger("myqueue-poison")] string message, TextWriter log)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connString);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("myqueue");
queue.CreateIfNotExists();
var messageData = JsonConvert.SerializeObject(data, new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver() });
queue.AddMessage(new CloudQueueMessage(messageData));
}
As at (2018-09-11) version 1.4.1 of the Microsoft Azure Storage Explorer doesn’t have the ability to move messages from one Azure queue to another.
I blogged a simple solution to transfer poison messages back to the originating queue and thought it might save someone a few minutes. Obviously, you'll need to have fixed the error that caused the messages to end up in the poison message queue!
You’ll need to add a NuGet package reference to Microsoft.NET.Sdk.Functions :
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Queue;
void Main()
{
const string queuename = "MyQueueName";
string storageAccountString = "xxxxxx";
RetryPoisonMesssages(storageAccountString, queuename);
}
private static int RetryPoisonMesssages(string storageAccountString, string queuename)
{
CloudQueue targetqueue = GetCloudQueueRef(storageAccountString, queuename);
CloudQueue poisonqueue = GetCloudQueueRef(storageAccountString, queuename + "-poison");
int count = 0;
while (true)
{
var msg = poisonqueue.GetMessage();
if (msg == null)
break;
poisonqueue.DeleteMessage(msg);
targetqueue.AddMessage(msg);
count++;
}
return count;
}
private static CloudQueue GetCloudQueueRef(string storageAccountString, string queuename)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageAccountString);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference(queuename);
return queue;
}
Azure Storage Explorer version 1.15.0 can now do this as of 2020. https://github.com/microsoft/AzureStorageExplorer/issues/1064
Essentially Azure Storage doesn't support moving messages from one queue to another. You would need to do this on your own.
One way to implement moving the messages from one queue to another is by dequeuing the messages from the source queue (by calling GetMessages), read the contents of the message and then creating a new message in the target queue. This you can do via using Storage Client Library.
One tool that comes to my mind for moving messages is Cerebrata Azure Management Studio(paid product with 15 days free trial). It has this functionality.
As at (2018-09-11) version 1.4.1 of the Microsoft Azure Storage Explorer doesn't support moving queue messages.
Here's an updated version of Mitch's answer, using the latest Microsoft.Azure.Storage.Queue package. Simply create a new .NET Console application, add the above-mentioned package to it, and replace the contents of Program.cs with the following:
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Queue;
using System.Threading.Tasks;
namespace PoisonMessageDequeuer
{
class Program
{
static async Task Main(string[] args)
{
const string queuename = "MyQueueName";
string storageAccountString = "xxx";
await RetryPoisonMesssages(storageAccountString, queuename);
}
private static async Task<int> RetryPoisonMesssages(string storageAccountString, string queuename)
{
var targetqueue = GetCloudQueueRef(storageAccountString, queuename);
var poisonqueue = GetCloudQueueRef(storageAccountString, queuename + "-poison");
var count = 0;
while (true)
{
var msg = await poisonqueue.GetMessageAsync();
if (msg == null)
break;
await poisonqueue.DeleteMessageAsync(msg);
await targetqueue.AddMessageAsync(msg);
count++;
}
return count;
}
private static CloudQueue GetCloudQueueRef(string storageAccountString, string queuename)
{
var storageAccount = CloudStorageAccount.Parse(storageAccountString);
var queueClient = storageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(queuename);
return queue;
}
}
}
It's still pretty slow if you're working with >1000 messages though, so I'd recommend looking into batch APIs for higher quantities.
Here's a python script you may find useful. You'll need to install azure-storage-queue
queueService = QueueService(connection_string = "YOUR CONNECTION STRING")
for queue in queueService.list_queues():
if "poison" in queue.name:
print(queue.name)
targetQueueName = queue.name.replace("-poison", "")
while queueService.peek_messages(queue.name):
for message in queueService.get_messages(queue.name, 32):
print(".", end="", flush=True)
queueService.put_message(targetQueueName, message.content)
queueService.delete_message(queue.name, message.id, message.pop_receipt)
I just had to do this again and took the time to update my snipped to the new storage SDKs. See post at https://www.bokio.se/engineering-blog/how-to-re-run-the-poison-queue-in-azure-webjobs/ for more info.
Here is the code I used
using Azure.Storage.Queues;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace AzureQueueTransfer
{
internal class Program
{
// Need Read, Update & Process (full url, can create in storage explorer)
private const string sourceQueueSAS = "";
// Need Add (full url, can create in storage explorer)
private const string targetQueueSAS = "";
private static async Task Main(string[] args)
{
var sourceQueue = new QueueClient(new Uri(sourceQueueSAS));
var targetQueue = new QueueClient(new Uri(targetQueueSAS));
var queuedAny = true;
while (queuedAny)
{
Thread.Sleep(30000); // Sleep to make sure we dont build too much backlog so we can process new messages on higher prio than old ones
queuedAny = false;
foreach (var message in sourceQueue.ReceiveMessages(maxMessages: 32).Value)
{
queuedAny = true;
var res = await targetQueue.SendMessageAsync(message.Body);
Console.WriteLine($"Transfered: {message.MessageId}");
await sourceQueue.DeleteMessageAsync(message.MessageId, message.PopReceipt);
}
Console.WriteLine($"Finished batch");
}
}
}
}
To anyone coming here looking for a Node equivalent of #MitchWheats answer using an Azure Function.
import AzureStorage from 'azure-storage'
import { Context, HttpRequest } from '#azure/functions'
import util from 'util'
const queueService = AzureStorage.createQueueService()
queueService.messageEncoder = new AzureStorage.QueueMessageEncoder.TextBase64QueueMessageEncoder()
const deleteMessage = util.promisify(queueService.deleteMessage).bind(queueService)
const createMessage = util.promisify(queueService.createMessage).bind(queueService)
const getMessage = util.promisify(queueService.getMessage).bind(queueService)
export async function run (context: Context, req: HttpRequest): Promise<void> {
try {
const poisonQueue = (req.query.queue || (req.body && req.body.queue));
const targetQueue = poisonQueue.split('-')[0]
let count = 0
while (true) {
const message = await getMessage(poisonQueue)
if (!message) { break; }
if (message.messageText && message.messageId && message.popReceipt) {
await createMessage(targetQueue, message.messageText)
await deleteMessage(poisonQueue, message.messageId, message.popReceipt)
}
count++
}
context.res = {
body: `Replayed ${count} messages from ${poisonQueue} on ${targetQueue}`
};
} catch (e) {
context.res = { status: 500 }
}
}
To use the function you need to you provide connection information for the storage account used for your storage queues. This is provided as environment variables. Either you provide AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_ACCESS_KEY, or AZURE_STORAGE_CONNECTION_STRING. More on this is available in the Azure Storage SDK docs.
Also wrote a few lines about it in this Medium article
Updated python based on Jon Canning's answer:
from azure.storage.queue import QueueServiceClient
queueService = QueueServiceClient.from_connection_string(conn_str="DefaultEndpointsProtocol=https;AccountName=<account>;AccountKey=<key>;EndpointSuffix=core.windows.net")
for queue in queueService.list_queues():
if "poison" in queue.name:
print(queue.name)
targetQueueName = queue.name.replace("-poison", "")
queue = queueService.get_queue_client(queue=queue.name)
targetQueue = queueService.get_queue_client(queue=targetQueueName)
while queue.peek_messages() :
messages = queue.receive_messages()
for msg in messages:
targetQueue.send_message(msg.content)
queue.delete_message(msg)
As Mikael Eliasson noted, the code in IGx89 answer is broken because
AddMessageAsync will overwrite some info on the message and then
DeleteMessagAsync will give a 404. The better solution is to copy the
values into a new message for AddMessageAsync
Please see enhanced version of RetryPoisonMesssages with an ability to specify only list of messages(instead of all in a queue) and allow to copy messages instead of move them.
It also logs success/failure for each message.
/// <param name="storageAccountString"></param>
/// <param name="queuename"></param>
/// <param name="idsToMove">If not null, only messages with listed IDs will be moved/copied</param>
/// <param name="deleteFromPoisonQueue">if false, messages will be copied; if true, they will be moved
///Warning: if queue is big, keeping deleteFromPoisonQueue=false can cause the same row
///from poisonqueue to be copied more than once(the reason is not found yet)</param>
/// <returns></returns>
private static async Task<int> RetryPoisonMesssages(string storageAccountString, string queuename, string[] idsToMove=null, bool deleteFromPoisonQueue=false)
{
var targetqueue = GetCloudQueueRef(storageAccountString, queuename);
var poisonQueueName = queuename + "-poison";
var poisonqueue = GetCloudQueueRef(storageAccountString, poisonQueueName);
var count = 0;
while (true)
{
var msg = await poisonqueue.GetMessageAsync();
if (msg == null)
{
Console.WriteLine("No more messages in a queue " + poisonQueueName);
break;
}
string action = "";
try
{
if (idsToMove == null || idsToMove.Contains(msg.Id))
{
var msgToAdd = msg;
if (deleteFromPoisonQueue)
{
//The reason is that AddMessageAsync will overwrite some info on the message and then DeleteMessagAsync will give a 404.
//The better solution is to copy the values into a new message for AddMessageAsync
msgToAdd = new CloudQueueMessage(msg.AsBytes);
}
action = "adding";
await targetqueue.AddMessageAsync(msgToAdd);
Console.WriteLine(action + " message ID " + msg.Id);
if (deleteFromPoisonQueue)
{
action = "deleting";
await poisonqueue.DeleteMessageAsync(msg);
}
Console.WriteLine(action + " message ID " + msg.Id);
}
}
catch (Exception ex)
{
Console.WriteLine("Error encountered when "+ action + " " + ex.Message + " at message ID " + msg.Id);
}
count++;
}
return count;
}
I want to make Twitter Sentiment Analysis Windows Phone Application.
the application works by retrieving all the related tweets based on what query terms that users entered. for example, if I enter "Windows Phone" in input search box, the results will show all the tweet that contains "windows phone" terms.
here's the code (that I get from Arik Poznanski's Blog)
/// <summary>
/// Searches the specified search text.
/// </summary>
/// <param name="searchText">The search text.</param>
/// <param name="onSearchCompleted">The on search completed.</param>
/// <param name="onError">The on error.</param>
public static void Search(string searchText, Action<IEnumerable<Twit>> onSearchCompleted = null, Action<Exception> onError = null, Action onFinally = null)
{
WebClient webClient = new WebClient();
// register on download complete event
webClient.OpenReadCompleted += delegate(object sender, OpenReadCompletedEventArgs e)
{
try
{
// report error
if (e.Error != null)
{
if (onError != null)
{
onError(e.Error);
}
return;
}
// convert json result to model
Stream stream = e.Result;
DataContractJsonSerializer dataContractJsonSerializer = new DataContractJsonSerializer(typeof(TwitterResults));
TwitterResults twitterResults = (TwitterResults)dataContractJsonSerializer.ReadObject(stream);
App thisApp = Application.Current as App;
thisApp.klasifikasi = new Klasifikasi();
foreach (Twit Tweet in twitterResults.results)
{
try
{
thisApp.klasifikasi.UploadData(Tweet); //requesting
break;
}
finally
{
// notify finally callback
if (onFinally != null)
{
onFinally();
}
}
}
//thisApp.klasifikasi.UploadDatas(twitterResults.results);
//thisApp.PositiveTweetModel = new PositiveTweetModel("Positive", twitterResults.results);
// notify completed callback
if (onSearchCompleted != null)
{
onSearchCompleted(twitterResults.results);
/// Divide the list here
}
}
finally
{
// notify finally callback
if (onFinally != null)
{
onFinally();
}
}
};
string encodedSearchText = HttpUtility.UrlEncode(searchText);
webClient.OpenReadAsync(new Uri(string.Format(TwitterSearchQuery, encodedSearchText)));
}
and to call the method
TwitterService.Search(
text,
(items) => { PositiveList.ItemsSource = items; },
(exception) => { MessageBox.Show(exception.Message); },
null
);
to upload POST Data into the API
public void UploadData(Twit tweetPerSend)
{
if (NetworkInterface.GetIsNetworkAvailable())
{
chatterbox.Headers[HttpRequestHeader.ContentType] = "application/x-www- form-urlencoded";
chatterbox.Headers["X-Mashape-Authorization"] = "MXBxYmptdjhlbzVnanJnYndicXNpN2NwdWlvMWE1OjA0YTljMWJjMDg4MzVkYWY2YmIzMzczZWFkNDlmYWRkNDYzNGU5NmI=";
var Uri = new Uri("https://chatterboxco-sentiment-analysis-for-social-media---nokia.p.mashape.com/sentiment/current/classify_text/");
StringBuilder postData = new StringBuilder();
postData.AppendFormat("{0}={1}", "lang", HttpUtility.UrlEncode("en"));
postData.AppendFormat("&{0}={1}", "text", HttpUtility.UrlEncode(tweetPerSend.DecodedText));
postData.AppendFormat("&{0}={1}", "exclude", HttpUtility.UrlEncode("is")); // disesuaikan
postData.AppendFormat("&{0}={1}", "detectlang", HttpUtility.UrlEncode("0"));
chatterbox.UploadStringAsync(Uri, "POST", postData.ToString());
chatterbox.UploadStringCompleted += new UploadStringCompletedEventHandler(chatterbox_UploadStringCompleted);
}
}
void chatterbox_UploadStringCompleted(object sender, UploadStringCompletedEventArgs e)
{
var chatterbox = sender as WebClient;
chatterbox.UploadStringCompleted -= chatterbox_UploadStringCompleted;
string response = string.Empty;
if (!e.Cancelled)
{
response = HttpUtility.UrlDecode(e.Result);
nilaiKlasifikasi = ParsingHasil(response);
MessageBox.Show(nilaiKlasifikasi.ToString()); //just testing
//textBlock1.Text = response;
}
}
private double ParsingHasil(String response)
{
var result = Regex.Match(#response, #"(?<=""value"": )(-?\d+(\.\d+)?)(?=,|$)");
Debug.WriteLine(result);
double hasil = Convert.ToDouble(result.ToString());
//return Convert.ToInt32(result);
return hasil;
}
However, there isn't only 1 tweet to retrieve, there'll be many tweets, so the main problem is, after I retrieve all the tweet and request the result to the API, I get this error "WebClient does not support concurrent I/O operations"
Does anyone know how to solve this problem?
any help would be appreciated
You'll have to execute UploadStringAsync synchronously one at a time. (i.e. chain execution of the next UploadStringAsync in the UploadStringCompleted handler.
Or, create a new WebClient for each UploadStringAsync.