I have a crawl function that also checks whether the content contains the param. If it contains I will write that to the database. How can I use the following code as a Read Job for the spring batch?
public void crawl(String baseUrl, String url, String postgresParam) {
if (!urls.contains(url) && url.startsWith(baseUrl)) {
// System.out.println(">> count: " + count + " [" + url + "]");
urls.add(url);
try {
Connection connection = Jsoup.connect(url).userAgent(USER_AGENT);
Document htmlDocument = connection.get();
Elements linksOnPage = htmlDocument.select("a[href]");
bodyContent = htmlDocument.body().text();
String title = htmlDocument.title();
searchParameters(url, title);
// count++;
for (Element link : linksOnPage) {
crawl(baseUrl, link.absUrl("href"), postgresParam);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
private void searchParameters(String URL, String title) {
for (String param : postgresParamArray) {
if (bodyContent.toLowerCase().contains(param.toLowerCase())) {
System.out.println(">>>>>> Found: " + " [" + param + "]" + " [" + URL + "]" + " [" + title + "]");
}
}
}
Related
this is my service code
public class Worker : BackgroundService
{
public bool isRegister { get; set; }
public bool checkIp { get; set; }
public long timePass { get; set; }
public Worker()
{
}
public override Task StartAsync(CancellationToken cancellationToken)
{
return base.StartAsync(cancellationToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
if (isRegister == false)
registerSelect();
if (checkIp == true)
{
checkIp = false;
await SecurityConfig.cacheServices?.BuildServiceProvider()?.GetService<IBlockFirewallIpService>().RegisterInFirewall();
}
timePass += 1000;
if (timePass % 60000 == 0)
await SecurityConfig.cacheServices?.BuildServiceProvider()?.GetService<IBlockFirewallIpService>().RegisterInFirewall();
await Task.Delay(1000, stoppingToken);
}
}
public void registerSelect()
{
isRegister = true;
using (SqlConnection conn = new SqlConnection(GetDbConnection()))
{
conn.Open();
SqlDependency.Start(GetDbConnection());
string commandText = "SELECT [Ip1],[Ip2] ,[Ip3] ,[Ip4] FROM dbo.BlockFirewallIps where IsRead is null";
using (SqlCommand cmd = new SqlCommand(commandText, conn))
{
SqlDependency dependency = new SqlDependency(cmd);
dependency.OnChange += new OnChangeEventHandler(OnDependencyChange);
cmd.ExecuteNonQuery();
}
conn.Close();
}
}
void OnDependencyChange(object sender, SqlNotificationEventArgs e)
{
if (e.Info == SqlNotificationInfo.Insert)
checkIp = true;
SqlDependency temp = sender as SqlDependency;
if (temp != null)
temp.OnChange -= new OnChangeEventHandler(OnDependencyChange);
registerSelect();
}
private string GetDbConnection()
{
return GlobalConfig.Configuration["ConnectionStrings:DefaultConnection"];
}
}
and this is my IBlockFirewallIpService.RegisterInFirewall() code
public async Task RegisterInFirewall()
{
var allBlockIps = await db.BlockFirewallIps.Where(t => t.IsRead == null).ToListAsync();
foreach (var ip in allBlockIps)
{
BanIP("OjeFirTCP" + ip.Ip1 + "_" + ip.Ip2 + "_" + ip.Ip3 + "_" + ip.Ip4, ip.Ip1 + "." + ip.Ip2 + "." + ip.Ip3 + "." + ip.Ip4, "Any", "TCP");
BanIP("OjeFirUDP" + ip.Ip1 + "_" + ip.Ip2 + "_" + ip.Ip3 + "_" + ip.Ip4, ip.Ip1 + "." + ip.Ip2 + "." + ip.Ip3 + "." + ip.Ip4, "Any", "UDP");
ip.IsRead = true;
db.SaveChanges();
}
}
void BanIP(string RuleName, string IPAddress, string Port, string Protocol)
{
if (OperatingSystem.IsWindows())
{
if (!string.IsNullOrEmpty(RuleName) && !string.IsNullOrEmpty(IPAddress) && !string.IsNullOrEmpty(Port) && !string.IsNullOrEmpty(Protocol) && new WindowsPrincipal(WindowsIdentity.GetCurrent()).IsInRole(WindowsBuiltInRole.Administrator))
{
using (Process RunCmd = new Process())
{
RunCmd.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
RunCmd.StartInfo.FileName = "cmd.exe";
RunCmd.StartInfo.Arguments = "/C netsh advfirewall firewall add rule name=\"" + RuleName + "\" dir=in action=block remoteip=" + IPAddress + " remoteport=" + Port + " protocol=" + Protocol;
RunCmd.Start();
}
}
}
}
this is progeram.cs
IHost host = Host.CreateDefaultBuilder(args)
.UseWindowsService(options =>
{
options.ServiceName = "OjeFirewall";
})
.ConfigureServices((hostContext, services) =>
{
GlobalConfig.Configuration = hostContext.Configuration;
services.AddScoped<IHttpContextAccessor, FakeIHttpContextAccessor>();
SecurityConfig.Config(services);
services.AddHostedService<Worker>();
})
.Build();
await host.RunAsync();
this is SecurityConfig.Config codes
services.AddDbContext<SecurityDBContext>(options =>
options.UseSqlServer(GlobalConfig.Configuration["ConnectionStrings:DefaultConnection"],
b => b.UseQuerySplittingBehavior(QuerySplittingBehavior.SingleQuery))
, ServiceLifetime.Singleton
);
services.AddSingleton<IIpLimitationWhiteListService, IpLimitationWhiteListService>();
services.AddSingleton<IIpLimitationBlackListService, IpLimitationBlackListService>();
services.AddSingleton<IFileAccessRoleService, FileAccessRoleService>();
services.AddSingleton<IRoleService, RoleService>();
services.AddSingleton<IBlockClientConfigService, BlockClientConfigService>();
services.AddSingleton<IBlockAutoIpService, BlockAutoIpService>();
services.AddSingleton<IBlockFirewallIpService, BlockFirewallIpService>();
the problem :
this code using too much memory after 3 day
starting ram usage is 20mb after first call (OnDependencyChange) it use 47mb
after 3 day it use 178mb
where i am doing wrong ?!
i found problem
await SecurityConfig.cacheServices?.BuildServiceProvider()?.GetService<IBlockFirewallIpService>().RegisterInFirewall();
this line code was the problem after change it to normal injection ram usage is stable now, but i can not understand whay !?
I have created a simple Spring-boot application with two entities as Company.java and User.java. These two has #OneToMany relationship. And I have a created a test file for generating typescript file with printing those two entity's attributes. Here is the my test case.
#Inject
RepositoryRestMvcConfiguration configuration;
#Test
public void getEndPoints() {
configuration.resourceMappings().forEach(c -> {
String className = c.getDomainType().getName();
try {
Class<?> entityClass = Class.forName(className);
Field[] fields = entityClass.getDeclaredFields();
File tsClassDir = new File("data/tsClass");
File tsClass = new File(tsClassDir, entityClass.getSimpleName() + ".ts");
if (!tsClass.getParentFile().exists()) {
tsClass.getParentFile().mkdirs();
}
tsClass.createNewFile();
String code = "export interface " + entityClass.getSimpleName() + "{\n";
for (Field field : fields) {
try {
NotNull notNullAnnotation = field.getDeclaredAnnotation(NotNull.class);
Class<?> filedClass = Class.forName(field.getType().getName());
if (notNullAnnotation == null){
code += "\t" + field.getName() + "?: " + filedClass.getSimpleName().trim() + ";" + "\n";
}else{
code += "\t" + field.getName() + ": " + filedClass.getSimpleName().trim() + ";" + "\n";
}
} catch (Exception e) {
// TODO: handle exception
}
// System.err.println(field.getName());
}
code += "}";
Files.write(tsClass.toPath(), code.getBytes());
System.err.println(code);
} catch (Exception e) {
// TODO: handle exception
}
});
}
After test run I got the result given below.
export interface User{
userName: String;
password: String;
email: String;
company?: Company;
}
export interface Company{
name: String;
email: String;
users?: Set;
}
But I need to print that Company and User has #OneToMany relationship in the typescript file. How do I do that?
jedis src code MasterListener method run:
j.subscribe(new JedisPubSub() {
#Override
public void onMessage(String channel, String message) {
log.fine("Sentinel " + host + ":" + port + " published: " + message + ".");
String[] switchMasterMsg = message.split(" ");
if (switchMasterMsg.length > 3) {
if (masterName.equals(switchMasterMsg[0])) {
initPool(toHostAndPort(Arrays.asList(switchMasterMsg[3], switchMasterMsg[4])));
} else {
log.fine("Ignoring message on +switch-master for master name "
+ switchMasterMsg[0] + ", our master name is " + masterName);
}
} else {
log.severe("Invalid message received on Sentinel " + host + ":" + port
+ " on channel +switch-master: " + message);
}
}
}, "+switch-master");
if there has three Sentinels,so created three MasterListener.When failover happens,jedis client will initPool three times for each MasterListener.
The question is：why not just initPool once? When sentinel objective offline master,then jedis client receive message to re-initPool?
I think synchronization is needed at this line "if (!master.equals(currentHostMaster)) {" to prevent create connection pool multiple times.
public class JedisSentinelPool extends JedisPoolAbstract {
private volatile HostAndPort currentHostMaster;
private void initPool(HostAndPort master) {
if (!master.equals(currentHostMaster)) {
currentHostMaster = master;
if (factory == null) {
}
I am using Spring AOP for logging wherein I want to log input/output of all methods present in package. I have written following pointcut for target package.
#Pointcut("within(com.mypackage.model.*)")
public void allmethods(){};
My logging method is as below.
#Before("allmethods()")
public void LoggingAdviceBefore(JoinPoint joinPoint)
{
StringBuffer logMessage = new StringBuffer();
if(joinPoint != null && joinPoint.getTarget()!=null && joinPoint.getTarget().getClass()!=null)
{
logMessage.append(joinPoint.getTarget().getClass().getName());
logMessage.append(".");
logMessage.append(joinPoint.getSignature().getName());
logMessage.append("(");
// append args
Object[] args = joinPoint.getArgs();
for (int i = 0; i < args.length; i++) {
logMessage.append(args[i]).append(",");
}
if (args.length > 0) {
logMessage.deleteCharAt(logMessage.length() - 1);
}
logMessage.append(")");
log.info(logMessage.toString());
}
}
The code is working fine.
My problem is, even if I do some simple operations like, populating an array list within my code, even that information is getting logged. I don't want such information to be logged.
I want to log inputs only for the methods that I had written in the classes present in target package & not for the code written inside those methods. How do I achieve this?
You can use the below code which I had written months back to understand SNMP framework implementation, it prints i/o of all the methods in package and subpackage, you can remove irrelevant classes and modify according to your needs, if required.
#Aspect
public class Snmp4JProfiler {
private static final Logger LOG = LoggerFactory.getLogger(Snmp4JProfiler.class);
#Pointcut("execution (* org.snmp4j.Snmp.*(..))")
public void allSnmpServices() {
}
#Pointcut("execution (* org.snmp4j.security.U*.*(..))")
public void allUSMServices() {
}
#Around("allUSMServices() || allSnmpServices()")
public Object executionTimeOfService(ProceedingJoinPoint pjp) throws Throwable {
MethodSignature methodSignature = (MethodSignature) pjp.getSignature();
String className = pjp.getSignature().getDeclaringTypeName();
final String methodName = methodSignature.getName();
String methodArgs = "";
for (Object obj : pjp.getArgs()) {
if (obj == null) {
methodArgs += "no args or null,";
} else if (obj instanceof UsmTimeEntry) {
UsmTimeEntry timeEntry = (UsmTimeEntry) obj;
methodArgs += obj.toString() + "[" + timeEntry.getEngineBoots() + "," + timeEntry.getLatestReceivedTime() + ","
+ timeEntry.getTimeDiff() + "," + timeEntry.getEngineID() + "]";
} else if (obj instanceof Object[]) {
methodArgs += obj.toString() + " " + Arrays.toString((Object[]) obj);
} else {
methodArgs += obj;
}
}
LOG.info("Start of method#" + methodName + " #class#" + className + " #args#" + methodArgs);
try {
Object output = pjp.proceed();
String rtValues = "";
if (output == null) {
rtValues += "no args or null,";
} else if (output instanceof UsmTimeEntry) {
UsmTimeEntry timeEntry = (UsmTimeEntry) output;
rtValues += output.toString() + "[" + timeEntry.getEngineBoots() + "," + timeEntry.getLatestReceivedTime() + ","
+ timeEntry.getTimeDiff() + "," + timeEntry.getEngineID() + "]";
} else if (output instanceof Object[]) {
rtValues += output.toString() + " " + Arrays.toString((Object[]) output);
} else {
rtValues += output;
}
LOG.info("End of method#" + methodName + " #class#" + className + " #return#" + rtValues);
return output;
} catch (Exception ex) {
LOG.info("End of method#" + methodName + " #class#" + className + " #error#" + ex.getMessage());
throw ex;
}
}
}
There
I’m using the EF code first to persist the data by following the sequence: filter => remove => add, run the attached sample(two threads run conrrrently), sometime I noticed there was no existing record return after the filtering, sometime there are two records after the filtering, what I thought/expected is - every time the filter should have one existing record return. Also, there are some exceptions raised up during saving the change.
As I known, EF by default uses the Read Committed isolation level to execute the transaction, I think that means during the filtering, the shared lock is put on the resource, but why I can observe that there is not existing record or two existing records after the filtering, the remove and add operation together should be an atomic operation, right? If I’m right, there should be only and just one record after filtering.
Is there anything I missed? How to handle this case correctly?
Please help.
Another question:
Use the LastUpdated column as the concurrency token, how to the following case correctly:
1. If the entity in the database is newer than the entity in the context, start another thread to archive the entity to the history database.
2. If the entity in the database is old than the entity in the context, retry the saving to overwrite the entity in the database.
Am i right to use the code below to handle the case:
internal void SaveChangesEx()
{
bool saveFailed;
do
{
saveFailed = false;
try
{
base.SaveChanges();
}
catch (DbUpdateConcurrencyException ex)
{
saveFailed = false;
ex.Entries.ToList().ForEach(entry =>
{
if (entry.State == System.Data.EntityState.Deleted)
{
var current = base.Set<Customer>().FirstOrDefault();
Customer rfqInDb = (Customer)entry.GetDatabaseValues();
// If the newer one aready exists, start the archive, otherwise, retry by reloading the entity from DB and marking the state as deleted
if (current.LastUpdated < customerInDb.LastUpdated)
{
using (var archiver = new CustomerArchiveDBContext())
{
archiver.RFQS.Add(current);
archiver.SaveChangesEx();
}
saveFailed = false;
}
else
{
//Refresh the context and retry
entry.Reload();
entry.State = System.Data.EntityState.Deleted;
}
}
});
}
} while (saveFailed);
}
Script: ========
CREATE TABLE [dbo].[Customers](
[FirstName] [nvarchar](20) NOT NULL,
[LastName] [nvarchar](60) NULL,
[Company] [nvarchar](250) NULL,
[Telephone] [nvarchar](20) NULL,
[LastUpdated] [datetime] NULL
) ON [PRIMARY]
Code ========
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Transactions;
using System.Data.Entity.Validation;
using System.Threading.Tasks;
using System.Threading;
using System.Data.Entity.ModelConfiguration;
using System.Data.Entity;
using System.Data.Entity.ModelConfiguration.Conventions;
namespace EFOptimisticConcurrency
{
public abstract class Entity
{
public int Id { get; set; }
}
public class Customer
: Entity
{
public string FirstName { get; set; }
/// <summary>
/// Get or set the surname of this customer
/// </summary>
public string LastName { get; set; }
/// <summary>
/// Get or set the telephone
/// </summary>
public string Telephone { get; set; }
/// <summary>
/// Get or set the company name
/// </summary>
public string Company { get; set; }
public DateTime LastUpdated { get; set; }
}
class CustomerEntityConfiguration
: EntityTypeConfiguration<Customer>
{
/// <summary>
/// Create a new instance of customer entity configuration
/// </summary>
public CustomerEntityConfiguration()
{
//configure keys and properties
this.HasKey(c => c.FirstName);
this.Ignore(c => c.Id);
this.Property(c => c.FirstName)
.HasMaxLength(20)
.IsRequired();
this.Property(c => c.LastName)
.HasMaxLength(60)
.IsRequired();
this.Property(c => c.Company)
.HasMaxLength(250);
//this.Property(c => c.LastUpdated).IsConcurrencyToken();
this.Property(c => c.Telephone)
.HasMaxLength(20);
this.ToTable("Customers");
}
}
public class CustomerContext : DbContext
{
public DbSet<Customer> Customers { get; set; }
protected override void OnModelCreating(DbModelBuilder modelBuilder)
{
modelBuilder.Conventions.Remove<OneToManyCascadeDeleteConvention>();
modelBuilder.Configurations.Add(new CustomerEntityConfiguration()); ;
}
}
public class Program
{
public static volatile int showStopper = 0;
static void Main(string[] args)
{
var color = Console.ForegroundColor;
EntityFrameworkProfiler.Initialize();
Task.Factory.StartNew(() =>
{
while (true)
{
Customer customer = new Customer();
customer.FirstName = "FirstName";
customer.LastName = "Last " + new Random().Next(0, 10000).ToString();
customer.Telephone = "686868";
customer.Company = "MyCity";
customer.LastUpdated = DateTime.Now;
if (showStopper == 2)
{
Console.ReadLine();
showStopper = 0;
}
try
{
Console.WriteLine("Start the Store => " + customer.LastName + " , " + customer.LastUpdated.ToString());
{
int i = 0;
using (var customerConext = new CustomerContext())
{
Console.WriteLine("Start the filter 1 => " + customer.Telephone + " , " + customer.LastUpdated.ToString());
var matched = customerConext.Customers.Where(c => c.Telephone == "686868" && c.LastUpdated < customer.LastUpdated);
foreach (var hit in matched)
{
i++;
customerConext.Customers.Remove(hit);
}
if (i == 2)
{
Console.WriteLine("1 - 2 exist, has the problem now");
showStopper = 2;
}
else if (i == 0)
{
Console.WriteLine("1 - 0 exist, has the problem now");
showStopper = 2;
}
Console.WriteLine("Start Adding 1 => " + customer.LastName + " , " + customer.LastUpdated.ToString());
try
{
customerConext.Customers.Add(customer);
customerConext.SaveChanges();
Console.WriteLine("SaveChanges 1 => " + customer.LastName + " , " + customer.LastUpdated.ToString());
}
catch (Exception ex)
{
Console.WriteLine("Exception 1 : " + ex.Message + " => " + customer.LastName + " , " + customer.LastUpdated);
if (ex.InnerException != null)
{
Console.WriteLine("Inner Exception 2 : " + ex.InnerException.Message + " => " + customer.LastName + " , " + customer.LastUpdated);
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Exception 1 " + ex.Message);
if (ex.InnerException != null)
{
Console.WriteLine(ex.InnerException.Message);
}
showStopper = 2;
}
}
});
Thread.Sleep(10000);
Task.Factory.StartNew(() =>
{
while (true)
{
Console.ForegroundColor = color;
try
{
Customer customer = new Customer();
customer.FirstName = "FirstName";
customer.LastName = "Last " + new Random().Next(0, 10000).ToString();
customer.Telephone = "686868";
customer.Company = "MyCity2";
customer.LastUpdated = DateTime.Now;
if (showStopper == 3)
{
Console.ReadLine();
showStopper = 0;
}
Console.WriteLine("Start the store 2 => " + customer.LastName + " , " + customer.LastUpdated.ToString());
{
int i = 0;
using (var customerConext = new CustomerContext())
{
Console.WriteLine("Start the filter 2 => " + customer.Telephone + " , " + customer.LastUpdated.ToString());
var matched = customerConext.Customers.Where(c => c.Telephone == "686868" && c.LastUpdated < customer.LastUpdated);
foreach (var hit in matched)
{
i++;
customerConext.Customers.Remove(hit);
}
Console.WriteLine("Start Adding 2 => " + customer.LastName + " , " + customer.LastUpdated.ToString());
try
{
customerConext.Customers.Add(customer);
customerConext.SaveChanges();
Console.WriteLine("SaveChanges 2 => " + customer.LastName + " , " + customer.LastUpdated.ToString());
}
catch (Exception ex)
{
Console.WriteLine("Exception 2 : " + ex.Message + " => " + customer.LastName + " , " + customer.LastUpdated);
if (ex.InnerException != null)
{
Console.WriteLine("Inner Exception 2 : " + ex.InnerException.Message + " => " + customer.LastName + " , " + customer.LastUpdated);
}
showStopper = 2;
}
}
if (i == 2)
{
Console.WriteLine("1 - 2 exist, has the problem now");
showStopper = 2;
}
else if (i == 0)
{
Console.WriteLine("1 - 0 exist, has the problem now");
showStopper = 2;
}
}
}
catch (Exception ex)
{
Console.WriteLine("Exception 2 " + ex.Message);
if (ex.InnerException != null)
{
Console.WriteLine(ex.InnerException.Message);
}
}
}
});
Console.WriteLine("PRESS ANY KEY TO END");
Console.ReadLine();
}
}
}