Inmemoryoutbox not work with a masstransit jobconsumers jobcontext - masstransit

I am using a JobConsumer to select a batch of items to be processed in a different process. For each item to process a new message is publish. I am using an inmemoryoutbox but the messages are published directly to rabbitmq. Actually I want all these messages to be collected in a memoryoutbox and all send after the loop has succeeded.
When I debug my code the PublishEndpointProvider has a Masstransit.Middleware.InMemoryOutbix.InMemoryOutboxPublishEndpointProvider. But the messages are send directly to RabbitMq.
I am using Masstransit version 8.0.1.
Any idea what is missing in the configuration?
configuration
services.AddMassTransit(mt =>
{
mt.AddDelayedMessageScheduler();
mt.AddConsumersFromNamespaceContaining<RunConsumer>();
mt.UsingRabbitMq((context, cfg) =>
{
cfg.UseDelayedMessageScheduler();
cfg.Host(rabbitMqConfig.Host, host =>
{
host.Username(rabbitMqConfig.Username);
host.Password(rabbitMqConfig.Password);
host.Heartbeat(rabbitMqConfig.Heartbeat);
});
cfg.UseGlobalRetryPolicy();
cfg.UseInMemoryOutbox();
var options = new ServiceInstanceOptions()
.EnableJobServiceEndpoints();
cfg.ServiceInstance(options, instance =>
{
instance.ConfigureJobServiceEndpoints(x =>
{
x.JobServiceStateEndpointName = "JobType";
x.JobServiceJobAttemptStateEndpointName = "JobAttempt";
x.JobServiceJobStateEndpointName = "Job";
});
instance.ConfigureEndpoints(context);
});
});
});
services.AddHostedService<MassTransitConsoleHostedService>();
return services;
Consumer definition
public class RunConsumerDefinition :
ConsumerDefinition<RunConsumer>
{
private IBusRegistrationContext context;
public ProlongeerPeriodeConsumerDefinition(IBusRegistrationContext context)
{
this.context = context;
EndpointName = 'MyQueue';
ConcurrentMessageLimit = 1;
}
protected override void ConfigureConsumer(
IReceiveEndpointConfigurator endpointConfigurator,
IConsumerConfigurator<RunConsumer> consumerConfigurator)
{
consumerConfigurator.Options<JobOptions<RunConsumer>>(options => options
.SetJobTimeout(TimeSpan.FromMinutes(15))
.SetConcurrentJobLimit(2));
endpointConfigurator.UseMessageScope(context);
endpointConfigurator.UseInMemoryOutbox();
}
}
RunConsumer
public async Task Run(JobContext<Run> context)
{
foreach (var index in Enumerable.Range(1, 7))
{
var command = new ProcessCommand();
await context.Publish<ProcessCommand>(command, context.CancellationToken);
}
}

Job Consumers don't technically run as a consumer, they're run separately with a JobContext<T>. As such, they don't use any of the retry, outbox, or other filters from the pipeline.

Related

Trying to create the database using ABP framework while DB Migration

I have inserted connectionstring in "[AbpTenantConnectionStrings]" table for same tenant id.
My objective is to create both the DB and migrage the DB
DB
while running below code only "Default Db is creating", can anyone let me know how to create and migrate other DB as well. what changes need to be done in the below code.
var tenants = await _tenantRepository.GetListAsync(includeDetails: true);
var migratedDatabaseSchemas = new HashSet<string>();
foreach (var tenant in tenants)
{
using (_currentTenant.Change(tenant.Id))
{
if (tenant.ConnectionStrings.Any())
{
var tenantConnectionStrings = tenant.ConnectionStrings
.Select(x => x.Value)
.ToList();
if (!migratedDatabaseSchemas.IsSupersetOf(tenantConnectionStrings))
{
await MigrateDatabaseSchemaAsync(tenant);
migratedDatabaseSchemas.AddIfNotContains(tenantConnectionStrings);
}
}
await SeedDataAsync(tenant);
}
After the below changes able to migrate or create the Db based on the connection-string inserted in "AbpTenantConnectionStrings":
public async Task MigrateAsync()
{
var initialMigrationAdded = AddInitialMigrationIfNotExist();
//var str = (await _connectionResolver.ResolveAsync("Default1"));
if (initialMigrationAdded)
{
return;
}
Logger.LogInformation("Started database migrations...");
await MigrateDatabaseSchemaAsync();
await SeedDataAsync();
Logger.LogInformation($"Successfully completed host database migrations.");
var tenants = await _tenantRepository.GetListAsync(includeDetails: true);
var migratedDatabaseSchemas = new HashSet<string>();
foreach (var tenant in tenants)
{
using (_currentTenant.Change(tenant.Id))
{
if (tenant.ConnectionStrings.Any())
{
var tenantConnectionStrings = tenant.ConnectionStrings
.Select(x => x.Value)
.ToList();
tenantConnectionStrings.ForEach(async s =>
{
await MigrateDatabaseSchemaAsync(s);
// migratedDatabaseSchemas.AddIfNotContains(s);
});
//if (!migratedDatabaseSchemas.IsSupersetOf(tenantConnectionStrings))
//{
// await MigrateDatabaseSchemaAsync(tenant);
// migratedDatabaseSchemas.AddIfNotContains(tenantConnectionStrings);
//}
}
await SeedDataAsync(tenant);
}
Logger.LogInformation($"Successfully completed {tenant.Name} tenant database migrations.");
}
Logger.LogInformation("Successfully completed all database migrations.");
Logger.LogInformation("You can safely end this process...");
}
private async Task MigrateDatabaseSchemaAsync(string str)
{
//Logger.LogInformation(
// $"Migrating schema for {(tenant == null ? "host" : tenant.Name + " tenant")} database...");
foreach (var migrator in _dbSchemaMigrators)
{
await migrator.MigrateAsync(str);
}
}
public interface IBookStoreDbSchemaMigrator
{
Task MigrateAsync();
Task MigrateAsync(string con);
}
public class EntityFrameworkCoreBookStoreDbSchemaMigrator
: IBookStoreDbSchemaMigrator, ITransientDependency
{
private readonly IServiceProvider _serviceProvider;
public EntityFrameworkCoreBookStoreDbSchemaMigrator(
IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public async Task MigrateAsync()
{
/* We intentionally resolving the BookStoreMigrationsDbContext
* from IServiceProvider (instead of directly injecting it)
* to properly get the connection string of the current tenant in the
* current scope.
*/
await _serviceProvider.GetRequiredService<BookStoreMigrationsDbContext>()
.Database
.MigrateAsync();
}
public async Task MigrateAsync(string con)
{
var context = _serviceProvider
.GetRequiredService<BookStoreMigrationsDbContext>();
context.Database.SetConnectionString(con);
context.Database.Migrate();
}
}
Let me know if anyone has suggestions or a better approach.

How to cancel BackgroundJob or let a newjob don't retry?

my case similar the example, a user presses a 'report' button to start a long-running reporting job. You add this job to the queue and send the report's result to your user via email when it's completed. But, I don't need retries job execution when the job failed. I want job stop and send notification. And can cancel on service.
public void CancelAllMyJob()
{
var jobs = backgroundJobStore.GetWaitingJobsAsync(int.MaxValue).Result.Where(o => o.JobType.Contains(nameof(MyJob))).ToList();
if (jobs.Count > 0)
{
foreach (BackgroundJobInfo job in jobs)
{
backgroundJobStore.DeleteAsync(job).Wait();
}
}
}
public async Task StartMyJob(MyJobInput input)
{
MyJobArgs args = new MyJobArgs
{
TenantId = AbpSession.TenantId,
UserId = AbpSession.UserId.Value,
};
var id = await backgroundJobManager.EnqueueAsync<MyJob, MyJobArgs >(args);
(await backgroundJobStore.GetAsync(Convert.ToInt64(id))).TryCount = 0;
}
public async Task StartMyJob(MyJobInput input)
{
MyJobArgs args = new MyJobArgs
{
TenantId = AbpSession.TenantId,
UserId = AbpSession.UserId.Value,
};
await Task.Run(() =>
{
using (var uow = UnitOfWorkManager.Begin(TransactionScopeOption.RequiresNew))
{
myJob.Execute(args);
}
});
}
It's work, thanks!

Getting Masstransit to use scoped lifetime

I'm having an issue with getting the same instance of the IIdentityService(my own class) or IServiceProvider in my consumer observer. I have an identity service where I set the credentials for the user, which is used later in the consumer pipeline.
I have tried the code below along with other code and configuration changes.
_serviceProvider.GetRequiredService<IIdentityService>()
_serviceProvider.CreateScope()
var consumerScopeProvider = _serviceProvider.GetRequiredService<IConsumerScopeProvider>();
using (var scope = consumerScopeProvider.GetScope(context))
{
// this next line of code is where we must access the payload
// using a container specific interface to get access to the
// scoped IServiceProvider
var serviceScope = scope.Context.GetPayload<IServiceScope>();
var serviceProviderScoped = serviceScope.ServiceProvider;
IIdentityService identityService = _serviceProvider.GetRequiredService<IIdentityService>();
}
// Also tried this as Scoped
services.AddTransient<CustomConsumer>();
var busControl = Bus.Factory.CreateUsingRabbitMq(cfg =>
{
var host = cfg.Host(new Uri(busConfiguration.Address), h =>
{
h.Username(busConfiguration.Username);
h.Password(busConfiguration.Password);
});
cfg.ReceiveEndpoint(host, "queue_1", endpointCfg => ConfigureConsumers(endpointCfg, provider));
cfg.UseServiceScope(provider);
});
busControl.ConnectConsumeObserver(new ConsumeObserver(provider));
public class ConsumeObserver : IConsumeObserver
{
private readonly IServiceProvider _serviceProvider;
public ConsumeObserver(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
Task IConsumeObserver.PreConsume<T>(ConsumeContext<T> context)
{
var identityMessage = (IIdentityMessage)context.Message;
if (identityMessage == null)
{
return Task.CompletedTask;
}
// Here I get a "Cannot resolve scoped service '' from root provider." error
IIdentityService identityService = _serviceProvider.GetRequiredService<IIdentityService>();
var task = identityService.SetIdentityAsync(identityMessage.Identity.TenantIdentifier, identityMessage.Identity.UserIdentifier);
// This gets a different instance of the IIdentityService.cs service.
//IIdentityService identityService = _serviceProvider.CreateScope().ServiceProvider.GetRequiredService<IIdentityService>();
// called before the consumer's Consume method is called
return task;
}
Task IConsumeObserver.PostConsume<T>(ConsumeContext<T> context)
{
// called after the consumer's Consume method is called
// if an exception was thrown, the ConsumeFault method is called instead
return TaskUtil.Completed;
}
Task IConsumeObserver.ConsumeFault<T>(ConsumeContext<T> context, Exception exception)
{
// called if the consumer's Consume method throws an exception
return TaskUtil.Completed;
}
}
// This is where I need to identity credentials
services.AddScoped<UserContext>((provider =>
{
// This gets a different instance of IIdentityService
var identityService = provider.GetRequiredService<IIdentityService>();
var contextProvider = provider.GetRequiredService<IContextProvider>();
var identity = identityService.GetIdentity();
return contextProvider.GetContext(identity.UserId);
}));
var task = identityService.SetIdentityAsync(identityMessage.Identity.TenantIdentifier, identityMessage.Identity.UserIdentifier);
The setting of the identity above should be retrievable below.
var identity = identityService.GetIdentity();
What I get is null reference as the service provider is of a different instance.
Can anyone tell me how to get the same instance of the service provider through out the consumer pipeline?

TcpListener handle of multiple clients

I created MyListener which will start listening (using TcpListener) on his own thread upon creation. the TcpListener should handle multiple clients so i am running inside infinte while and handle each client in special task.
this is my code:
public class MyListener
{
public event EventHandler<MessageEventArgs> MessageReceived;
public MyListener()
{
var thread = new Thread(Listen);
thread.Start();
}
private void Listen()
{
TcpListener server = null;
try
{
server = new TcpListener(IPAddress.Any, 8977);
server.Start();
while (true)
{
var client = server.AcceptTcpClient();
Task.Run(() =>
{
try
{
var msg = GetMessageFromClient(client);
MessageReceived?.Invoke(this, new MessageEventArgs { Message = msg });
}
catch (Exception)
{
}
finally
{
client.Close();
}
});
}
}
catch (Exception)
{
}
finally
{
if (server != null)
server.Stop();
}
}
private string GetMessageFromClient(TcpClient client)
{
var bytes = new byte[client.ReceiveBufferSize];
var stream = client.GetStream();
var i = stream.Read(bytes, 0, bytes.Length);
var message = Encoding.UTF8.GetString(bytes, 0, i);
return message;
}
}
here are my questions:
how can i ensure that the task handle the client will use the client i pass to it when i start the task and not different client (becuase after the task start we return to the AcceptTcpClient method and may get new client)
in my example and with multiple clients handled by the same method ("GetMessageFromClient") do i need to put some kind of locking on this
method?

What is the TPL equivalent of rx's Observable.FromEventPattern?

In rx you can write :
var oe = Observable.FromEventPattern<SqlNotificationEventArgs>(sqlDep, "OnChange");
and then subscribe to the observable to convert the OnChange event on the sqlDep object into an observable.
Similarily, how can you create a Task from a C# event using the Task Parallel Library ?
EDIT: clarification
The solution pointed by Drew and then written explicitely by user375487 works for a single event. As soon as the task finished ... well it is finished.
The observable event is able to trigger again at any time. It is can be seen as an observable stream. A kind of ISourceBlock in the TPL Dataflow. But in the doc http://msdn.microsoft.com/en-us/library/hh228603(v=vs.110).aspx there is no example of ISourceBlock.
I eventually found a forum post explaining how to do that: http://social.msdn.microsoft.com/Forums/en/tpldataflow/thread/a10c4cb6-868e-41c5-b8cf-d122b514db0e
public static ISourceBlock CreateSourceBlock(
Action,Action,Action,ISourceBlock> executor)
{
var bb = new BufferBlock();
executor(t => bb.Post(t), () => bb.Complete(), e => bb.Fault(e), bb);
return bb;
}
//Remark the async delegate which defers the subscription to the hot source.
var sourceBlock = CreateSourceBlock<SomeArgs>(async (post, complete, fault, bb) =>
{
var eventHandlerToSource = (s,args) => post(args);
publisher.OnEvent += eventHandlerToSource;
bb.Complete.ContinueWith(_ => publisher.OnEvent -= eventHandlerToSource);
});
I've not tryed the above code. There may be a mismatch between the async delegate and the definition of CreateSourceBlock.
There is no direct equivalent for the Event Asynchronous Pattern (EAP) baked into the TPL. What you need to do is using a TaskCompletionSource<T> that you signal yourself in the event handler. Check out this section on MSDN for an example of what that would look like which uses WebClient::DownloadStringAsync to demonstrate the pattern.
You can use TaskCompletionSource.
public static class TaskFromEvent
{
public static Task<TArgs> Create<TArgs>(object obj, string eventName)
where TArgs : EventArgs
{
var completionSource = new TaskCompletionSource<TArgs>();
EventHandler<TArgs> handler = null;
handler = new EventHandler<TArgs>((sender, args) =>
{
completionSource.SetResult(args);
obj.GetType().GetEvent(eventName).RemoveEventHandler(obj, handler);
});
obj.GetType().GetEvent(eventName).AddEventHandler(obj, handler);
return completionSource.Task;
}
}
Example usage:
public class Publisher
{
public event EventHandler<EventArgs> Event;
public void FireEvent()
{
if (this.Event != null)
Event(this, new EventArgs());
}
}
class Program
{
static void Main(string[] args)
{
Publisher publisher = new Publisher();
var task = TaskFromEvent.Create<EventArgs>(publisher, "Event").ContinueWith(e => Console.WriteLine("The event has fired."));
publisher.FireEvent();
Console.ReadKey();
}
}
EDIT Based on your clarification, here is an example of how to achieve your goal with TPL DataFlow.
public class EventSource
{
public static ISourceBlock<TArgs> Create<TArgs>(object obj, string eventName)
where TArgs : EventArgs
{
BufferBlock<TArgs> buffer = new BufferBlock<TArgs>();
EventHandler<TArgs> handler = null;
handler = new EventHandler<TArgs>((sender, args) =>
{
buffer.Post(args);
});
buffer.Completion.ContinueWith(c =>
{
Console.WriteLine("Unsubscribed from event");
obj.GetType().GetEvent(eventName).RemoveEventHandler(obj, handler);
});
obj.GetType().GetEvent(eventName).AddEventHandler(obj, handler);
return buffer;
}
}
public class Publisher
{
public event EventHandler<EventArgs> Event;
public void FireEvent()
{
if (this.Event != null)
Event(this, new EventArgs());
}
}
class Program
{
static void Main(string[] args)
{
var publisher = new Publisher();
var source = EventSource.Create<EventArgs>(publisher, "Event");
source.LinkTo(new ActionBlock<EventArgs>(e => Console.WriteLine("New event!")));
Console.WriteLine("Type 'q' to exit");
char key = (char)0;
while (true)
{
key = Console.ReadKey().KeyChar;
Console.WriteLine();
if (key == 'q') break;
publisher.FireEvent();
}
source.Complete();
Console.ReadKey();
}
}

Resources