I working with gorp , i used Exec to UPDATE and INSERT .
updateDataQuery = " UPDATE time_tables SET line_code = ? , direction = ? , station_code = ? , " +
"train_number = ? , train_type = ? , dept_station_code = ? ," +
"arrv_station_code = ? , arrv_time = ? , dept_time = ? , time_table = ? " +
"WHERE line_code = ? AND station_code = ? AND train_number = ? AND time_table = ? "
result, err := db.Exec(updateDataQuery,Model.LineCode, Model.Direction, Model.StationCode,
Model.TrainNumber, Model.TrainType, Model.DeptStationCode,
Model.ArrvStationCode, Model.ArrvTime, Model.DeptTime, Model.TimeTable,
Model.LineCode, Model.StationCode, Model.TrainNumber, Model.TimeTable)
row, _ := result.RowsAffected()
if row == 0 {
// insert
}
first run : row = 0 => insert
second run : Model not change row = 0 => insert
how to replace data first run by data second run ?
Related
I am asking for your help.
I have an existing dag that cleans up airflow_db from our production airflow db. Now I need to adjust the code and add another connection to our test environment, I was advised to create a for cycle like
for conn_id in {"prod_db", "test_db"}:
with DAG( ... conn_id = conn_id, ...)
But I am not sure how to do that really, as I did the change the syntax is incorrect now.
Do you have any example/hint how to do that?
for conn_id in {"prod_db","test_db"}:
with DAG(
conn_id = conn_id,
dag_id="airflow_db_cleanup",
default_args={
"owner": "our team,
"email": mail_mail_mail,
"email_on_failure": True,
"email_on_retry": False,
"start_date": datetime(2022, 1, 1, 0, 0, 0),
"retries": 1,
"retry_delay": timedelta(minutes=1),
},
schedule_interval=timedelta(days=1),
max_active_runs=1,
) as dag,
dag.doc_md = doc
etc-then various functions that create sessions follow.
The operator here:
def get_max_db_entry_age_in_days(dag_run_conf):
max_db_entry_age_in_days = None
default_max_db_entry_age_in_days = int(Variable.get("max_db_entry_age_in_days", 90))
if dag_run_conf:
max_db_entry_age_in_days = dag_run_conf.get("maxDBEntryAgeInDays", None)
logging.info("maxDBEntryAgeInDays from dag_run.conf: " + str(dag_run_conf))
if max_db_entry_age_in_days is None:
logging.info(
"maxDBEntryAgeInDays conf variable isn't included. Using Default '"
+ str(default_max_db_entry_age_in_days)
+ "'"
)
max_db_entry_age_in_days = default_max_db_entry_age_in_days
return max_db_entry_age_in_days
def cleanup_function(**context):
dag_run = context.get("dag_run")
dag_run_conf = getattr(dag_run, "conf", None)
max_db_entry_age_in_days = get_max_db_entry_age_in_days(dag_run_conf)
execution_date = context.get("execution_date")
max_date = execution_date + timedelta(-max_db_entry_age_in_days)
dry_run = Variable.get("dry_run", "True") == "True"
airflow_db_model = context["params"].get("airflow_db_model")
age_check_column = context["params"].get("age_check_column")
keep_last_run = context["params"].get("keep_last_run")
dag_id = context["params"].get("dag_id")
cascade_delete = context["params"].get("cascade_delete")
with create_session(conn_id) as session:
logging.info("Configurations:")
logging.info("max_date: " + str(max_date))
logging.info("session: " + str(session))
logging.info("airflow_db_model: " + str(airflow_db_model))
logging.info("age_check_column: " + str(age_check_column))
logging.info("keep_last_run: " + str(keep_last_run))
logging.info("dag_id: " + str(dag_id))
logging.info("")
logging.info("Running Cleanup Process...")
query = get_main_query(
session, airflow_db_model, age_check_column, keep_last_run, dag_id, max_date
)
entries_to_delete_count = query.count()
logging.info(f"Query : {query}")
logging.info(
f"Will be deleting {entries_to_delete_count} {airflow_db_model.__name__}"
)
logging.info(list(query.values(dag_id)))
if cascade_delete:
for child in cascade_delete:
child_model = child.get("child_model")
child_id = child.get("child_id")
parent_id = child.get("parent_id")
delete_query = get_cascade_delete_query(
session, query, child_model, child_id, parent_id
)
child_rows = delete_query.count()
logging.info(
f"Going to delete {child_rows} rows of {child_model.__name__}."
)
logging.info(list(delete_query.values(child_id)))
if not dry_run:
delete_query.delete(synchronize_session=False)
if dry_run:
logging.info("Dry run finished")
else:
logging.info("Performing main delete...")
query.delete(synchronize_session=False)
session.commit()
logging.info("Cleanup process finished")
for db_object in DATABASE_OBJECTS:
cleanup_op = PythonOperator(
task_id="cleanup_" + str(db_object["airflow_db_model"].__name__),
python_callable=cleanup_function,
params=db_object,
provide_context=True,
dag=dag,
)
def remove_inactive_dags_from_ui(**context):
max_db_age_inactive_dags = int(Variable.get("max_db_age_inactive_dags", 7))
dry_run = Variable.get("dry_run", "True") == "True"
threshold_date = context["execution_date"].subtract(days=max_db_age_inactive_dags)
with create_session(conn_id) as session:
query = (
session.query(DagModel)
.filter(
and_(
DagModel.is_active.is_(False),
DagModel.is_subdag.is_(False),
DagModel.last_parsed_time <= threshold_date,
)
)
.options(load_only(DagModel.dag_id))
)
inactive_dags = query.all()
logging.info(f"Will be deleted {query.count()} dags")
for inactive_dag in inactive_dags:
logging.info(f"Deleting dag {inactive_dag.dag_id} from UI")
if not dry_run:
delete_dag(inactive_dag.dag_id, session=session)
if dry_run:
logging.info("Dry run finished")
else:
session.commit()
logging.info("Cleanup process finished")
remove_inactive_dags_task = PythonOperator(
task_id="remove_inactive_dags_from_ui",
python_callable=remove_inactive_dags_from_ui,
provide_context=True,
dag=dag,
)
Creating a loop of DAGs or Tasks is bad practice inside Airflow, I'm not sure if it's even possible.
The correct approach would be to have two tasks that run the same DAG code, but passing a different connection through each time. You could even parallel these tasks up so both DBs are cleared down at the same time rather than one after another.
I tried:
table.selectionRange = table.ranges["C6:C7"];
Gets: Error -1728: Can't get object.
I am able to use the table object: Eg: table.rowCount();
Any ideas?
Note: Syntax 'table.selectionRange = table.ranges["C6:C7"];' was posted as a solution here: How to make a range in Numbers (iWork) using JXA
Added for further clarity:
Logger.logInfo("#Rows " + table.rowCount());
Logger.logInfo("Current Range " + table.selectionRange.name());
Logger.logInfo("#Cols " + table.columnCount());
table.selectionRange = table.ranges["C6:C7"];
Logger.logInfo("New Range " + table.selectionRange.name());
Gives:
/* 2018/02/02 # 19:36:27.020: Info : BlockPriceUpdateYPF: #Rows 34 */
/* 2018/02/02 # 19:36:27.023: Info : BlockPriceUpdateYPF: Current Range C5:G31 */
/* 2018/02/02 # 19:36:27.025: Info : BlockPriceUpdateYPF: #Cols 15 */
Result: Error -1728: Can't get object.
The syntax above is correct, but one needs both:
a valid reference to a currently open table, and
(to set a .selectionRange) GUI focus, obtained, for example, using the .activate() method.
(() => {
const
app = Application('Numbers'),
ws = app.windows,
w = ws.length > 0 ? (
ws.at(0)
) : undefined,
sheets = w ? w.document.sheets : [],
sheet = sheets.length > 0 ? (
sheets.at(0)
) : undefined,
tables = sheet ? sheet.tables : [],
table = tables.length > 0 ? (
tables.at(0)
) : undefined;
app.activate();
// READ the selection range (name property of table.selectionRange())
// return table ? (() => {
// const maybeSelection = table.selectionRange();
// return (typeof maybeSelection) === 'function' ? (
// maybeSelection.name()
// ) : 'No range selected'
// })() : 'No active table found in Numbers';
// or SET the selection:
return table ? (
table.selectionRange = table.ranges['D10:D12'],
'Range now selected: ' + table.selectionRange.name()
) : 'No active table';
})();
I'm trying to do a search through a table via LINQ and Entity Framework CORE. I've got 2 text boxes startdate and enddate and a radio button set of 3 options created, modified and both.
This is the code I've produced based on Google searches and tutorials
switch(radCreatedModifiedBoth) {
case "b":
if (!String.IsNullOrEmpty(startDate)) {
if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps => (
ps.CreatedDate >= Convert.ToDateTime(startDate + " 00:00:00") &&
ps.CreatedDate <= Convert.ToDateTime(endDate + " 23:59:59")
) || (
ps.ModifiedDate >= Convert.ToDateTime(startDate + " 00:00:00") &&
ps.ModifiedDate <= Convert.ToDateTime(endDate + " 23:59:59")
)
);
} else {
persons = persons.Where(ps => (
ps.CreatedDate >= Convert.ToDateTime(startDate + " 00:00:00")
||
ps.ModifiedDate >= Convert.ToDateTime(startDate + " 00:00:00")
)
);
}
} else if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps => (
ps.CreatedDate >= Convert.ToDateTime(endDate + " 23:59:59")
||
ps.ModifiedDate >= Convert.ToDateTime(endDate + " 23:59:59")
)
);
}
break;
case "c":
if (!String.IsNullOrEmpty(startDate)) {
if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps => (
ps.CreatedDate >= Convert.ToDateTime(startDate + " 00:00:00") &&
ps.CreatedDate <= Convert.ToDateTime(endDate + " 23:59:59")
)
);
} else {
persons = persons.Where(ps => (
ps.CreatedDate >= Convert.ToDateTime(startDate + " 00:00:00")
)
);
}
} else if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps <= (
ps.CreatedDate >= Convert.ToDateTime(endDate + " 23:59:59")
)
);
}
break;
case "m":
if (!String.IsNullOrEmpty(startDate)) {
if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps => (
ps.ModifiedDate >= Convert.ToDateTime(startDate + " 00:00:00") &&
ps.ModifiedDate <= Convert.ToDateTime(endDate + " 23:59:59")
)
);
} else {
persons = persons.Where(ps => (
ps.ModifiedDate >= Convert.ToDateTime(startDate + " 00:00:00")
)
);
}
} else if (!String.IsNullOrEmpty(endDate)) {
persons = persons.Where(ps <= (
ps.ModifiedDate >= Convert.ToDateTime(endDate + " 23:59:59")
)
);
}
break;
}
This code works but seems massively inefficient, not to mention adding the start and end time into the date as a string like this
startDate + " 00:00:00"
endDate + " 23:59:59"
just seems wrong. Is this the prescribed method or can anyone suggest a more efficient method preferably getting rid of the " 00:00:00"/" 23:59:59"
Thanks
You can simplify by pushing the tests to SQL - the conditional operator will be translated to SQL CASE WHEN since they aren't simple constants. Note that I assumed you have the endDate tests backwards in your code sample. Also you have a lot of repeated sub-expressions I consolidated to variables. Since you are using EF Core, there isn't a better way to handle date only comparisons then what you are using. In EF you can use a DbFunction, but it still isn't as good as converting your dates to the appropriate date+time so that indices can be used.
var hasStartDate = !String.IsNullOrEmpty(startDate);
var dtStartDate = hasStartDate ? Convert.ToDateTime(startDate + " 00:00:00") : DateTime.MinValue;
var hasEndDate = !String.IsNullOrEmpty(endDate);
var dtEndDate = hasEndDate ? Convert.ToDateTime(endDate + " 23:59:59") : DateTime.MinValue;
var chkCreatedDate = (radCreatedModifiedBoth == "b" || radCreatedModifiedBoth == "c");
var chkModifiedDate = (radCreatedModifiedBoth == "b" || radCreatedModifiedBoth == "m");
persons = persons.Where(ps => (chkCreatedDate ? (hasStartDate ? ps.CreatedDate >= dtStartDate : true) && (hasEndDate ? ps.CreatedDate <= dtEndDate : true) : true)
||
(chkModifiedDate ? (hasEndDate ? ps.ModifiedDate >= dtStartDate : true) && (hasEndDate ? ps.ModifiedDate <= dtEndDate : true) : true)
);
With the following query how to I change that I dont have two sets of fields in the select new I want the information going into one set of columns not having two and a type field to say if its a traineeevent or a cpd event ?
List<EmployeeCPDReportRecord> employeeCPDRecords = new List<EmployeeCPDReportRecord>();
string employeeName;
var q = from cpd in pamsEntities.EmployeeCPDs
from traineeEvent in pamsEntities.TrainingEventTrainees
join Employee e in pamsEntities.Employees on cpd.EmployeeID equals e.emp_no
join TrainingEventPart tEventPart in pamsEntities.TrainingEventParts on traineeEvent.TrainingEventPartId equals tEventPart.RecordId
where (cpd.EmployeeID == id) && (startDate >= cpd.StartDate && endDate <= cpd.EndDate) &&
(traineeEvent.EmployeeId == id)
&& (traineeEvent.TraineeStatus == 1 || traineeEvent.TraineeStatus == 2)
&& (tEventPart.CPDHours > 0 || tEventPart.CPDPoints > 0)
&& (cpd.CPDHours > 0 || cpd.CPDPoints > 0)
|| traineeEvent.StartDate >= startDate
|| traineeEvent.EndDate <= endDate
orderby cpd.StartDate
select new
{
surname = e.surname,
forname1 = e.forename1,
forname2 = e.forename2,
EmployeeID = cpd.EmployeeID,
StartDate = cpd.StartDate,
EndDate = cpd.EndDate,
CPDHours = cpd.CPDHours,
CPDPoints = cpd.CPDPoints,
Description = cpd.Description,
TrainingStartDate = tEventPart.StartDate,
TrainingEndDate = tEventPart.EndDate,
TrainingCPDHours = tEventPart.CPDHours,
TrainingCPDPoints = tEventPart.CPDPoints,
TrainingEventDescription = tEventPart.Description
};
if (q != null)
{
Array.ForEach(q.ToArray(), i =>
{
if (ContextBase.encryptionEnabled)
employeeName = ContextBase.Decrypt(i.surname) + ", " + ContextBase.Decrypt(i.forname1) + " " + ContextBase.Decrypt(i.forname2);
else
employeeName = i.surname + ", " + i.forname1 + " " + i.forname2;
if (i.TrainingStartDate != new DateTime(1900, 1, 1))
employeeCPDRecords.Add(new EmployeeCPDReportRecord(employeeName, Convert.ToDateTime(i.StartDate), Convert.ToDateTime(i.EndDate), Convert.ToDecimal(i.CPDHours), Convert.ToDecimal(i.CPDPoints), i.Description,i.t,i.EndDate,Convert.ToDecimal(i.CPDHours),Convert.ToDecimal(i.CPDPoints),i.Description,"L&D"));
else
employeeCPDRecords.Add(new EmployeeCPDReportRecord(employeeName, Convert.ToDateTime(i.StartDate), Convert.ToDateTime(i.EndDate), Convert.ToDecimal(i.CPDHours), Convert.ToDecimal(i.CPDPoints), i.Description, i.StartDate, i.EndDate, Convert.ToDecimal(i.CPDHours), Convert.ToDecimal(i.CPDPoints), i.Description, "Employee CPD"));
});
}
Use this code
List<EmployeeCPDReportRecord> employeeCPDRecords = new List<EmployeeCPDReportRecord>();
var q = ( from cpd in pamsEntities.EmployeeCPDs
from traineeEvent in pamsEntities.TrainingEventTrainees
join Employee e in pamsEntities.Employees on cpd.EmployeeID equals e.emp_no
join TrainingEventPart tEventPart in pamsEntities.TrainingEventParts on traineeEvent.TrainingEventPartId equals tEventPart.RecordId
where (cpd.EmployeeID == id) && (startDate >= cpd.StartDate && endDate <= cpd.EndDate) &&
(traineeEvent.EmployeeId == id)
&& (traineeEvent.TraineeStatus == 1 || traineeEvent.TraineeStatus == 2)
&& (tEventPart.CPDHours > 0 || tEventPart.CPDPoints > 0)
&& (cpd.CPDHours > 0 || cpd.CPDPoints > 0)
|| traineeEvent.StartDate >= startDate
|| traineeEvent.EndDate <= endDate
orderby cpd.StartDate
select new EmployeeCPDReportRecord
{
YourEmployeColumnName=(ContextBase.encryptionEnabled==true?ContextBase.Decrypt(e.surname) + ", " + ContextBase.Decrypt(e.forname1) + " " + ContextBase.Decrypt(e.forname2):e.surname + ", " + e.forname1 + " " + e.forname2),
YourEmployeeCPDColumnName=(i.TrainingStartDate !=new DateTime(1900, 1, 1)?"L&D":"Employee CPD")
surname = e.surname,
forname1 = e.forename1,
forname2 = e.forename2,
EmployeeID = cpd.EmployeeID,
StartDate = cpd.StartDate,
EndDate = cpd.EndDate,
CPDHours = cpd.CPDHours,
CPDPoints = cpd.CPDPoints,
Description = cpd.Description,
TrainingStartDate = tEventPart.StartDate,
TrainingEndDate = tEventPart.EndDate,
TrainingCPDHours = tEventPart.CPDHours,
TrainingCPDPoints = tEventPart.CPDPoints,
TrainingEventDescription = tEventPart.Description
}).ToList<EmployeeCPDReportRecord>();
I'd like an elegant way to concatenate several columns together using LINQ, but using the + operator or concat() when any of the columns are NULL results in NULL for the value after concatenation.
Is there anything similar to concat() that handles NULL differently, or am I thinking about this in the incorrect way?
Any help is appreciated!
Here is the code I am using:
List<CustomObject> objects = (
from obj in ObjectTable
where obj.Id == Id
select new CustomObject()
{
EnteredBy = obj.EnteredBy,
EntryDate = obj.EntryDate,
WorknoteText =
obj.VchWorkNote1 +
obj.VchWorkNote2 +
obj.VchWorkNote3 +
obj.VchWorkNote4 +
obj.VchWorkNote5 +
obj.VchWorkNote6 +
obj.VchWorkNote7 +
obj.VchWorkNote8 +
obj.VchWorkNote9 +
obj.VchWorkNote10 +
obj.VchWorkNote11 +
obj.VchWorkNote12 +
obj.VchWorkNote13 +
obj.VchWorkNote14 +
obj.VchWorkNote15 +
obj.VchWorkNote16 +
obj.VchWorkNote17 +
obj.VchWorkNote18 +
obj.VchWorkNote19 +
obj.VchWorkNote20
}).ToList();
One option is to use the null coalescing operator:
List<CustomObject> objects = (from o in ObjectTable
where o.Id == Id
select new CustomObject(){
EnteredBy = o.EnteredBy,
EntryDate = o.EntryDate,
WorknoteText =
(o.VchWorkNote1 ?? "") +
(o.VchWorkNote2 ?? "") +
(o.VchWorkNote3 ?? "") +
(o.VchWorkNote4 ?? "") +
...
(o.VchWorkNote20 ?? "")
}).ToList();
Hopefully the generated SQL will use an appropriate translation.
You can use the ?? operator this way :
...
(object.VchWorkNote5 ?? "") +
(object.VchWorkNote6 ?? "") +
(object.VchWorkNote7 ?? "") +
...
How about (object.VchWorkNote1 ?? "") +.....
Can you add a new column to your database? Something like "Keywords" or "FullText"
Define it to have a calculation, that calculation is basically "ISNULL(<Field1>, '') + ISNULL(<Field2>, '')" etc.
Make sure to mark it as persisted, so it doesn't have to calculate each time.
Then you just need to pull down that one field from your table.
there's probably a cleaner way, but first thing that comes to mind would be a quick null to zero length string conversion:
List<CustomObject> objects = (from object in ObjectTable
where object.Id == Id
select new CustomObject(){
EnteredBy = object.EnteredBy,
EntryDate = object.EntryDate,
WorknoteText =
object.VchWorkNote1 ?? "" +
object.VchWorkNote2 ?? "" +
object.VchWorkNote3 ?? "" +
object.VchWorkNote4 ?? "" +
object.VchWorkNote5 ?? "" +
object.VchWorkNote6 ?? "" +
object.VchWorkNote7 ?? "" +
object.VchWorkNote8 ?? "" +
object.VchWorkNote9 ?? "" +
object.VchWorkNote10 ?? "" +
object.VchWorkNote11 ?? "" +
object.VchWorkNote12 ?? "" +
object.VchWorkNote13 ?? "" +
object.VchWorkNote14 ?? "" +
object.VchWorkNote15 ?? "" +
object.VchWorkNote16 ?? "" +
object.VchWorkNote17 ?? "" +
object.VchWorkNote18 ?? "" +
object.VchWorkNote19 ?? "" +
object.VchWorkNote20 ?? ""
}).ToList();