SQLAlchemy custom boolean - oracle

I have an Oracle database with boolean logic implemented this way:
MYCOL NUMBER(1,0)
0 = False
-1 or 1 = True
NULL = None
With a default SQLAlchemy boolean column, -1 table value doesn't match False or True.
Is it possible to create an SQLAlchemy custom type that maps such a column to a bool python attribute?
The difficulty relies in having multiple values associated with true.
Ex:
session.query(User).filter_by(active=True)
Should generate one of these queries:
select * from user where active != 0;
select * from user where active in (-1, 1);
select * from user where 1 = (CASE WHEN active != 0 THEN 1 ELSE active end);

This implementation seems to work for basic queries.
Case when expression seems to be the easiest solution because operator logic isn't altered.
class CustomBoolean(sqlalchemy.types.TypeDecorator):
"""0 = False
-1 or 1 = True
NULL = None
By default, SQLAlchemy only supports 0 and 1"""
impl = Boolean
cache_ok = True
# In select or group by clause, replace col with expression: (CASE WHEN col != 0 THEN 1 ELSE col end)
def column_expression(self, col):
expr = col.expr
if isinstance(expr.type, CustomBoolean):
res = sqlalchemy.case(
(type_coerce(expr, Integer()) != sqlalchemy.literal_column("0", type_=Integer), sqlalchemy.literal_column("1", type_=Integer)),
else_=type_coerce(expr, Integer())
)
return type_coerce(res, Boolean())
expr.type = Boolean()
return expr
# In where clause, replace col with expression: (CASE WHEN col != 0 THEN 1 ELSE col end)
class comparator_factory(Boolean.Comparator):
def operate(self, op, other, **kwargs):
op1 = self
if hasattr(op1, "__clause_element__"):
clause = op1.__clause_element__()
if isinstance(clause.type, CustomBoolean):
op1 = self.normalise(clause)
op2 = other
if hasattr(op2, "__clause_element__"):
clause = op2.__clause_element__()
if isinstance(clause.type, CustomBoolean):
op2 = self.normalise(clause)
return op(
op1, op2, **kwargs
)
#staticmethod
def normalise(expr):
res = sqlalchemy.case(
(type_coerce(expr, Integer()) != sqlalchemy.literal_column("0", type_=Integer), sqlalchemy.literal_column("1", type_=Integer)),
else_=type_coerce(expr, Integer())
)
return type_coerce(res, Boolean())

Related

Slow filtering of tableview - QSortFilterProxyModel - QAbstractTableModel

I'm working on an app which displays some member info in a table view. I have a filter function to search in the table:
![image of the ui file]
The content of the table loads really fast but the filtering is extremely slow.
Here is the code:
class PandasTableModel(QtCore.QAbstractTableModel):
def __init__(self, data):
super(PandasTableModel, self).__init__()
self._data = data
a=os.getcwd()
self.icons_folder = a + "/assets/icons/"
def data(self, index, role):
# value = self._data.iloc[index.row()][index.column()]
column_value = index.column()
if role == Qt.DisplayRole:
return self._data.iloc[index.row()][index.column()]
return str(value)
if role == Qt.DisplayRole:
if isinstance(value, datetime):
return value.strftime("%d-%m-%Y")
else:
return str(value)
if role == Qt.DecorationRole:
# if isinstance(value, datetime):
if column_value == 12 or column_value == 23 or column_value == 25:
return QtGui.QIcon(self.icons_folder + "calendar.png")
elif column_value == 14:
return QtGui.QIcon(self.icons_folder + "iphone-icon.png")
elif column_value == 13 or column_value == 15:
return QtGui.QIcon(self.icons_folder + "icon-phone.png")
elif column_value == 16:
return QtGui.QIcon(self.icons_folder + "email-icon.png")
def rowCount(self, index):
return self._data.shape[0]
def columnCount(self, index):
return self._data.shape[1]
def headerData(self, section, orientation, role):
# section is the index of the column/row.
if role == Qt.DisplayRole:
if orientation == Qt.Horizontal:
return str(self._data.columns[section])
if orientation == Qt.Vertical:
return str(self._data.index[section])
class MembersAdminWindow(QtWidgets.QMainWindow):
def __init__(self):
super(MembersAdminWindow, self).__init__()
self.selectionModel = None
self.ui = loadUi("uidesign/members/members2.ui", self)
self.default_pic = str(os.getcwd()) + "/assets/clipart/muaythai.png"
def LoadMemberdata(self):
self.ui.lblMemberID.clear()
self.ui.txtNickName.clear()
self.ui.txtName.clear()
self.ui.txtFirstName.clear()
self.ui.txtStreet.clear()
self.ui.txtHouseNumber.clear()
self.ui.txtCity.clear()
self.ui.txtPostalCode.clear()
self.ui.txtCountry.clear()
self.ui.txtNationality.clear()
self.ui.txtLanguage.clear()
self.ui.txtGender.clear()
self.ui.txtBirthDate.clear()
self.ui.txtFixedNumber.clear()
self.ui.txtMobileNumber.clear()
self.ui.txtBusinessNumber.clear()
self.ui.txtEmail.clear()
self.ui.txtPassword.clear()
self.ui.scrbMemberLevel.setValue(0)
self.ui.txtSportsType.clear()
self.ui.txtStatusMember.clear()
self.ui.txtDoctor.clear()
self.ui.txtSubscriptionType.clear()
self.ui.txtMemberStart.clear()
self.ui.txtInsuranceStart.clear()
self.ui.txtMemberInfo.clear()
self.ui.txtSaldo.clear()
self.ui.txtSaldo.hide()
self.ui.lblSaldo.hide()
self.ui.lblMemberCard.setPixmap(QtGui.QPixmap(self.default_pic))
start = timeit.default_timer()
print('Start LoadMemberdata\n')
self.ui.tableMembers.setEnabled(True)
folder = os.getcwd()
master_file = str(folder) + "/modules/ssh/downloads/master.pkl"
Member_data = pd.read_pickle(master_file)
df = pd.DataFrame(Member_data)
df.fillna('',inplace=True)
aantal_gebruikers = df.shape[0]
print(aantal_gebruikers, 'gebruikers zijn ingelezen')
self.model = PandasTableModel(df)
self.proxy_model = QSortFilterProxyModel()
self.proxy_model.setSourceModel(self.model)
self.ui.inputFilter.setText("")
self.ui.inputFilter.textChanged.connect(self.SetFilteredView)
filter_result = self.proxy_model.rowCount()
self.ui.lblTotalMembers.setText("(" + str(filter_result) + " gebruikers)")
self.ui.cmdNewMember.setEnabled(True)
self.ui.cmdSaveMember.setEnabled(False)
self.ui.cmdCancel.setEnabled(False)
self.ui.cmdChangeMember.setEnabled(False)
self.ui.cmdTakeMemberPicture.setEnabled(False)
self.ui.cmdMemberFaceId.setEnabled(False)
self.ui.lblSaldo.setEnabled(False)
self.ui.txtSaldo.setEnabled(False)
self.ui.lblSaldo.hide()
self.ui.txtSaldo.hide()
self.ui.lblMemberCard.setPixmap(QtGui.QPixmap(self.default_pic))
self.ui.groupMembersData.setEnabled(False)
print('\nEnd LoadMemberdata')
end = timeit.default_timer()
print("Process Time: ", (end - start))
def SetFilteredView(self):
print("Start set_filter")
filter_text = self.ui.inputFilter.text()
rows = self.ui.tableMembers.model().rowCount()
for row in range(rows):
self.filter_row(row, filter_text)
# self.proxy_model.setFilterFixedString(filter_text)
# filter_result = self.proxy_model.rowCount()
self.ui.lblTotalMembers.setText("(" + str(rows) + " gebruikers)")
print("Stop set_filter")
def filter_row(self, row, pattern):
print("Start filter_row")
if not pattern:
self.ui.tableMembers.setRowHidden(row, False)
return
model = self.ui.tableMembers.model()
columns = model.columnCount()
stringlist = []
# collect text from all columns into single string for searching
for c in range(columns):
mdx = model.index(row, c)
if mdx.isValid():
val = str(mdx.data(role=QtCore.Qt.DisplayRole)).lower()
stringlist.append(val)
# search for string
patterns = filter(None, [x.lower() for x in pattern.split(' ')])
results = all(any(x in y for y in stringlist) for x in patterns)
if results:
self.ui.tableMembers.setRowHidden(row, False)
else:
self.ui.tableMembers.setRowHidden(row, True)
print("Stop filter_row")
i noticed when i type in the filter text field that the function 'filter_row' is called about 2000 times so i think i have to look for the solution here ;-)
Can anyone give a hand ?
Cheers John

EF Core performance issue when using dynamic query using Expressions in .Net

Requirement : Select list of records from table for members whose ID and First name matches with records.
The number of members in the request may vary as per the request:
I have written following code to generate dynamic query using expressions.
private Expression<Func<TmpOncMemberAuth, bool>> CreateQueryExpression(List<MemberRequest> memberRequests)
{
var type = typeof(TmpOncMemberAuth);
// member =>
var memberParam = Expression.Parameter(type, "member");
var requests = memberRequests;
Expression selectLeft = null;
Expression filterExpression = null;
foreach (var member in requests)
{
var comparison = BuildSubQuery(member, memberParam, type);
if (selectLeft == null)
{
selectLeft = comparison;
filterExpression = selectLeft;
continue;
}
filterExpression =
Expression.Or(filterExpression, comparison);
}
return Expression.Lambda<Func<TmpOncMemberAuth, bool>>
(filterExpression, memberParam);
}
private Expression BuildSubQuery(MemberRequest member, ParameterExpression memberParam, Type type)
{
// LHS
// member.MemberID
Expression leftMemberID = Expression.Property(memberParam, type.GetProperty("MemberId"));
Expression leftMemberFirstName = Expression.Property(memberParam, type.GetProperty("MemberFirstName"));
// RHS
Expression requestMemberID = Expression.Constant(member.MemberID);
Expression requestFirstName = Expression.Constant(member.FirstName);
// member.MemberID == requestMemberID
Expression compareID =
Expression.Equal(leftMemberID, requestMemberID);
Expression compareFName =
Expression.Equal(leftMemberFirstName, requestFirstName);
// condition for a member
Expression comparison = Expression.AndAlso(compareID, compareFName);
return comparison;
}
// query call in the main
var whereQuery = CreateQueryExpression(memberData);
var memberAuthData = await FindAllAsyncFromTemp(whereQuery);
This generate a SQL query in the below format, which take a lot time
SELECT [#].[CaseID] AS [CaseId], [#].[MemberID] AS [MemberId], [#].[MemberFirstName], [#].[MemberLastName], [#].[MemberDOB]
FROM [#ONCMemberAuth] AS [#]
WHERE ((CASE
WHEN (([#].[MemberID] = '12345') AND ([#].[MemberFirstName] = 'James') ) THEN CAST(1 AS bit)
ELSE CAST(0 AS bit)
END | CASE
WHEN (([#].[MemberID] = '6789') AND ([#].[MemberFirstName] = 'WILLERS') ) THEN CAST(1 AS bit)
ELSE CAST(0 AS bit)
END)
) = CAST(1 AS bit)
I need to create SQL query like below which executes faster than the above one
SELECT [#].[CaseID] AS [CaseId], [#].[MemberID] AS [MemberId], [#].[MemberFirstName], [#].[MemberLastName], [#].[MemberDOB]
FROM [#ONCMemberAuth] AS [#]
WHERE ((([#].[MemberID] = '1234') AND ([#].[MemberFirstName] = 'James')) OR (([#].[MemberID] = '56789') AND ([#].[MemberFirstName] = 'WILLERS') ) )

Left outer join with multiple on conditions in LINQ

basically i want to check DestinationCustomerList with SA_CUSTOMER_ALLOC if any record found based on customerId and fieldId then return priority else priority = 1 , so i am using left outer join with entity but it's not working properly(returning only matching record and if SA_CUSTOMER_ALLOC doesn't have any record throwing "Object referenece error" how to handle this.
var Query = (from p in DestinationCustomerList
join p2 in Context.SA_CUSTOMER_ALLOC on new { f1 = p.CustomerId, f2 = p.FieldId } equals new { f1 = p2.CUSTOMER_ID, f2 = p2.FIELD_ID }
into temp
from x in temp.DefaultIfEmpty()
where x.TAG_ID == TagId && x.REGION_ID == RegionId
select new { p.CustomerId,p.FieldId, Priority = x == null ? 1 : x.PRIORITY }).ToList();
Instead of Priority = x == null ? 1 : x.PRIORITY
try Priority = p2 == null ? 1 : p2.PRIORITY
var Query = (from p in DestinationCustomerList
join p2 in Context.SA_CUSTOMER_ALLOC
on new
{ f1 = p.CustomerId, f2 = p.FieldId }
equals new
{ f1 = p2.CUSTOMER_ID, f2 = p2.FIELD_ID }
into temp
from x in temp.DefaultIfEmpty()
let priority = p2 == null ? 1 : p2.PRIORITY
where x.TAG_ID == TagId && x.REGION_ID == RegionId
select new { p.CustomerId,p.FieldId, Priority = priority }).ToList();

How can I deep-compare 2 Lua tables, which may or may not have tables as keys?

(Also posted on the Lua mailing list)
So I've been writing deep-copy algorithms, and I wanna test them to see if they work the way I want them to. While I do have access to the original->copy map, I want a general-purpose deep-compare algorithm that must be able to compare table keys (tables as keys?).
My deep-copy algorithm(s) are avaliable here: https://gist.github.com/SoniEx2/fc5d3614614e4e3fe131 (it's not very organized, but there are 3 of them, one uses recursive calls, the other uses a todo table, and the other simulates a call stack (in a very ugly but 5.1-compatible way))
Recursive version:
local function deep(inp,copies)
if type(inp) ~= "table" then
return inp
end
local out = {}
copies = (type(copies) == "table") and copies or {}
copies[inp] = out -- use normal assignment so we use copies' metatable (if any)
for key,value in next,inp do -- skip metatables by using next directly
-- we want a copy of the key and the value
-- if one is not available on the copies table, we have to make one
-- we can't do normal assignment here because metatabled copies tables might set metatables
-- out[copies[key] or deep(key,copies)]=copies[value] or deep(value,copies)
rawset(out,copies[key] or deep(key,copies),copies[value] or deep(value,copies))
end
return out
end
Edit: I found things like this which don't really handle tables as keys: http://snippets.luacode.org/snippets/Deep_Comparison_of_Two_Values_3 (Copy of snippet below)
function deepcompare(t1,t2,ignore_mt)
local ty1 = type(t1)
local ty2 = type(t2)
if ty1 ~= ty2 then return false end
-- non-table types can be directly compared
if ty1 ~= 'table' and ty2 ~= 'table' then return t1 == t2 end
-- as well as tables which have the metamethod __eq
local mt = getmetatable(t1)
if not ignore_mt and mt and mt.__eq then return t1 == t2 end
for k1,v1 in pairs(t1) do
local v2 = t2[k1]
if v2 == nil or not deepcompare(v1,v2) then return false end
end
for k2,v2 in pairs(t2) do
local v1 = t1[k2]
if v1 == nil or not deepcompare(v1,v2) then return false end
end
return true
end
Serializing is also not an option, as order of serialization is "random".
As others said, that depends a lot on your definition of equivalence.
If you want this to be true:
local t1 = {[{}] = {1}, [{}] = {2}}
local t2 = {[{}] = {1}, [{}] = {2}}
assert( table_eq(t1, t2) )
If you do, then each time the key in t1 is a table, you'll have to
check its equivalence with every table key in t2 and try them one by
one. This is a way to do it (metatable stuff left out for readability).
function table_eq(table1, table2)
local avoid_loops = {}
local function recurse(t1, t2)
-- compare value types
if type(t1) ~= type(t2) then return false end
-- Base case: compare simple values
if type(t1) ~= "table" then return t1 == t2 end
-- Now, on to tables.
-- First, let's avoid looping forever.
if avoid_loops[t1] then return avoid_loops[t1] == t2 end
avoid_loops[t1] = t2
-- Copy keys from t2
local t2keys = {}
local t2tablekeys = {}
for k, _ in pairs(t2) do
if type(k) == "table" then table.insert(t2tablekeys, k) end
t2keys[k] = true
end
-- Let's iterate keys from t1
for k1, v1 in pairs(t1) do
local v2 = t2[k1]
if type(k1) == "table" then
-- if key is a table, we need to find an equivalent one.
local ok = false
for i, tk in ipairs(t2tablekeys) do
if table_eq(k1, tk) and recurse(v1, t2[tk]) then
table.remove(t2tablekeys, i)
t2keys[tk] = nil
ok = true
break
end
end
if not ok then return false end
else
-- t1 has a key which t2 doesn't have, fail.
if v2 == nil then return false end
t2keys[k1] = nil
if not recurse(v1, v2) then return false end
end
end
-- if t2 has a key which t1 doesn't have, fail.
if next(t2keys) then return false end
return true
end
return recurse(table1, table2)
end
assert( table_eq({}, {}) )
assert( table_eq({1,2,3}, {1,2,3}) )
assert( table_eq({1,2,3, foo = "fighters"}, {["foo"] = "fighters", 1,2,3}) )
assert( table_eq({{{}}}, {{{}}}) )
assert( table_eq({[{}] = {1}, [{}] = {2}}, {[{}] = {1}, [{}] = {2}}) )
assert( table_eq({a = 1, [{}] = {}}, {[{}] = {}, a = 1}) )
assert( table_eq({a = 1, [{}] = {1}, [{}] = {2}}, {[{}] = {2}, a = 1, [{}] = {1}}) )
assert( not table_eq({1,2,3,4}, {1,2,3}) )
assert( not table_eq({1,2,3, foo = "fighters"}, {["foo"] = "bar", 1,2,3}) )
assert( not table_eq({{{}}}, {{{{}}}}) )
assert( not table_eq({[{}] = {1}, [{}] = {2}}, {[{}] = {1}, [{}] = {2}, [{}] = {3}}) )
assert( not table_eq({[{}] = {1}, [{}] = {2}}, {[{}] = {1}, [{}] = {3}}) )
Instead of direct comparison you may try to serialize each of the tables and compare the serialized results. There are several serializers that handle table as keys and can serialize deep and recursive structures. For example, you may try Serpent (available as a standalone module and also included in Mobdebug):
local dump = pcall(require, 'serpent') and require('serpent').dump
or pcall(require, 'mobdebug') and require('mobdebug').dump
or error("can't find serpent or mobdebug")
local a = dump({a = 1, [{}] = {}})
local b = dump({[{}] = {}, a = 1})
print(a==b)
This returns true for me (both Lua 5.1 and Lua 5.2). One of the caveats is that the serialization result depends on the order in which keys are serialized. The tables as key by default are sorted based on their tostring value, which means that the order depends on their allocation address and it's not difficult to come up with an example that fails under Lua 5.2:
local dump = pcall(require, 'serpent') and require('serpent').dump
or pcall(require, 'mobdebug') and require('mobdebug').dump
or error("can't find serpent or mobdebug")
local a = dump({a = 1, [{}] = {1}, [{}] = {2}})
local b = dump({[{}] = {2}, a = 1, [{}] = {1}})
print(a==b) --<-- `false` under Lua 5.2
One way to protect against this is to consistently represent tables in keys comparison; for example, instead of default tostring, you may want to serialize tables and their values and sort the keys based on that (serpent allows a custom handler for sortkeys), which would make the sorting more robust and the serialized results more stable.

LINQ attempts to truncate variable value to length of comparison

The where clause in the LINQ statement below causes the following error when 'logLevel' has a value of "Error"
Error message = An error occurred while executing the command definition. See the inner exception for details.
Inner exception = p_linq_2 : String truncation: max=3, len=5, value='Error'
If I change && (logLevel == "All" || logLevel == "Error") to && (logLevel == "Error") the error no longer occurs.
It seems that to do the comparison against "All", LINQ tries to truncate the value in 'logLevel' to 3 characters first?
What is going on here?
(I'm running MVC 4, SQL CE 4)
IQueryable<LogEvent> list = (from a in _context.ELMAH_Error
where a.TimeUtc >= start && a.TimeUtc <= end
&& (logLevel == "All" || logLevel == "Error")
select new LogEvent
{
IdType = "guid", Id = "",IdAsInteger = 0,IdAsGuid = a.ErrorId,LoggerProviderName = "Elmah",LogDate = a.TimeUtc,MachineName = a.Host,Message = a.Message,Type = a.Type,Level = "Error",Source = a.Source,StackTrace = ""
});

Resources