now = datetime.utcnow().replace(tzinfo=utc)
.annotate(
age=F(int((now - 'ended_at').total_seconds() / (60 * 60)))
I want to add logic like above in Django query.
basically i want to calculate "age" that's a reason need to perform operation ORM.
having large data and its taking time if I do this operation using for loop.
First define a Func to extract the number of seconds since the UNIX epoch.
from django.db.models import Func, IntegerField
class UnixTime (Func):
"""
Extract the number of seconds since January 1, 1970.
"""
arity = 1
output_field = IntegerField()
# for PostgreSQL
def as_sql(self, compiler, connection, **extra_context):
return super().as_sql(
compiler, connection,
template="EXTRACT(EPOCH FROM %(expressions)s)",
**extra_context)
def as_mysql(self, compiler, connection, **extra_context):
return super().as_sql(
compiler, connection,
template="UNIX_TIMESTAMP(%(expressions)s)",
**extra_context)
def as_sqlite(self, compiler, connection, **extra_context):
return super().as_sql(
compiler, connection,
template="CAST(strftime('%%%%s', %(expressions)s) AS INTEGER)",
**extra_context)
Then make a query like this:
from django.db.models import F
from django.db.models.functions import Now
YourObject.objects.annotate(
age=(UnixTime(Now()) - UnixTime(F('ended_at'))) / 3600
)
Related
I am training a VAE (using federated learning, but that is not so important) and wanted to keep the loss and train functions simple to exchange. The initial approach was to have a tf.function as loss function and a tf.function as train function as follows:
#tf.function
def kl_reconstruction_loss(model, model_input, beta):
x, y = model_input
mean, logvar = model.encode(x, y)
z = model.reparameterize(mean, logvar)
x_logit = model.decode(z, y)
cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)
reconstruction_loss = tf.reduce_mean(tf.reduce_sum(cross_ent, axis=[1, 2, 3]), axis=0)
kl_loss = tf.reduce_mean(0.5 * tf.reduce_sum(tf.exp(logvar) + tf.square(mean) - 1. - logvar, axis=-1), axis=0)
loss = reconstruction_loss + beta * kl_loss
return loss, kl_loss, reconstruction_loss
#tf.function
def train_fn(model: tf.keras.Model, batch, optimizer, kl_beta):
"""Trains the model on a single batch.
Args:
model: The VAE model.
batch: A batch of inputs [images, labels] for the vae.
optimizer: The optimizer to train the model.
beta: Weighting of KL loss
Returns:
The loss.
"""
def vae_loss():
"""Does the forward pass and computes losses for the generator."""
# N.B. The complete pass must be inside loss() for gradient tracing.
return kl_reconstruction_loss(model, batch, kl_beta)
with tf.GradientTape() as tape:
loss, kl_loss, rc_loss = vae_loss()
grads = tape.gradient(loss, model.trainable_variables)
grads_and_vars = zip(grads, model.trainable_variables)
optimizer.apply_gradients(grads_and_vars)
return loss
For my dataset this results in an epoch duration of approx. 25 seconds. However, since I have to call those functions directly in my code, I would have to enter different ones if I would want to try out different loss/train functions.
So, alternatively, I followed https://github.com/google-research/federated/tree/master/gans and wrapped the loss function in a class and the train function in another function. Now I have:
class VaeKlReconstructionLossFns(AbstractVaeLossFns):
#tf.function
def vae_loss(self, model, model_input, labels, global_round):
# KL Reconstruction loss
mean, logvar = model.encode(model_input, labels)
z = model.reparameterize(mean, logvar)
x_logit = model.decode(z, labels)
cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=model_input)
reconstruction_loss = tf.reduce_mean(tf.reduce_sum(cross_ent, axis=[1, 2, 3]), axis=0)
kl_loss = tf.reduce_mean(0.5 * tf.reduce_sum(tf.exp(logvar) + tf.square(mean) - 1. - logvar, axis=-1), axis=0)
loss = reconstruction_loss + self._get_beta(global_round) * kl_loss
if model.losses:
loss += tf.add_n(model.losses)
return loss, kl_loss, reconstruction_loss
def create_train_vae_fn(
vae_loss_fns: vae_losses.AbstractVaeLossFns,
vae_optimizer: tf.keras.optimizers.Optimizer):
"""Create a function that trains VAE, binding loss and optimizer.
Args:
vae_loss_fns: Instance of gan_losses.AbstractVAELossFns interface,
specifying the VAE training loss.
vae_optimizer: Optimizer for training the VAE.
Returns:
Function that executes one step of VAE training.
"""
# We check that the optimizer has not been used previously, which ensures
# that when it is bound the train fn isn't holding onto a different copy of
# the optimizer variables then the copy that is being exchanged b/w server and
# clients.
if vae_optimizer.variables():
raise ValueError(
'Expected vae_optimizer to not have been used previously, but '
'variables were already initialized.')
#tf.function
def train_vae_fn(model: tf.keras.Model,
model_inputs,
labels,
global_round,
new_optimizer_state=None):
"""Trains the model on a single batch.
Args:
model: The VAE model.
model_inputs: A batch of inputs (usually images) for the VAE.
labels: A batch of labels corresponding to the inputs.
global_round: The current glob al FL round for beta calculation
new_optimizer_state: A possible optimizer state to overwrite the current one with.
Returns:
The number of examples trained on.
The loss.
The updated optimizer state.
"""
def vae_loss():
"""Does the forward pass and computes losses for the generator."""
# N.B. The complete pass must be inside loss() for gradient tracing.
return vae_loss_fns.vae_loss(model, model_inputs, labels, global_round)
# Set optimizer vars
optimizer_state = get_optimizer_state(vae_optimizer)
if new_optimizer_state is not None:
# if optimizer is uninitialised, initialise vars
try:
tf.nest.assert_same_structure(optimizer_state, new_optimizer_state)
except ValueError:
initialize_optimizer_vars(vae_optimizer, model)
optimizer_state = get_optimizer_state(vae_optimizer)
tf.nest.assert_same_structure(optimizer_state, new_optimizer_state)
tf.nest.map_structure(lambda a, b: a.assign(b), optimizer_state, new_optimizer_state)
with tf.GradientTape() as tape:
loss, kl_loss, rc_loss = vae_loss()
grads = tape.gradient(loss, model.trainable_variables)
grads_and_vars = zip(grads, model.trainable_variables)
vae_optimizer.apply_gradients(grads_and_vars)
return tf.shape(model_inputs)[0], loss, optimizer_state
return train_vae_fn
This new formulation takes about 86 seconds per epoch.
I am struggling to understand why the second version performs so much worse than the first one. Does anyone have a good explanation for this?
Thanks in advance!
EDIT: My Tensorflow version is 2.5.0
Hi i'm having an issue with the transfer of data from one database to another. I created a list using field in a table on a msql db, used that list to query and oracle db table (using the initial list in the where statement to filter results) I then load the query results back into the msql db.
The program runs for the first few iterations but then errors out, with the following error (
Traceback (most recent call last):
File "C:/Users/1/PycharmProjects/DataExtracts/BuyerGroup.py", line 67, in
insertIntoMSDatabase(idString)
File "C:/Users/1/PycharmProjects/DataExtracts/BuyerGroup.py", line 48, in insertIntoMSDatabase
mycursor.executemany(sql, val)
pyodbc.ProgrammingError: The second parameter to executemany must not be empty.)
I can't seem to find and guidance online to troubleshoot this error message. I feel it may be a simple solution but I just can't get there...
# import libraries
import cx_Oracle
import pyodbc
import logging
import time
import re
import math
import numpy as np
logging.basicConfig(level=logging.DEBUG)
conn = pyodbc.connect('''Driver={SQL Server Native Client 11.0};
Server='servername';
Database='dbname';
Trusted_connection=yes;''')
b = conn.cursor()
dsn_tns = cx_Oracle.makedsn('Hostname', 'port', service_name='name')
conn1 = cx_Oracle.connect(user=r'uid', password='pwd', dsn=dsn_tns)
c = conn1.cursor()
beginTime = time.time()
bind = (b.execute('''select distinct field1
from [server].[db].[dbo].[table]'''))
print('MSQL table(s) queried, List Generated')
# formats ids for sql string
def surroundWithQuotes(id):
return "'" + re.sub(",|\s$", "", str(id)) + "'"
def insertIntoMSDatabase(idString):
osql = '''SELECT distinct field1, field2
FROM Database.Table
WHERE field2 is not null and field3 IN ({})'''.format(idString)
c.execute(osql)
claimsdata = c.fetchall()
print('Oracle table(s) queried, Data Pulled')
mycursor = conn.cursor()
sql = '''INSERT INTO [dbo].[tablename]
(
[fields1]
,[field2]
)
VALUES (?,?)'''
val = claimsdata
mycursor.executemany(sql, val)
conn.commit()
ids = []
formattedIdStrings = []
# adds all the ids found in bind to an iterable array
for row in bind:
ids.append(row[0])
# splits the ids[] array into multiple arrays < 1000 in length
batchedIds = np.array_split(ids, math.ceil(len(ids) / 1000))
# formats the value inside each batchedId to be a string
for batchedId in batchedIds:
formattedIdStrings.append(",".join(map(surroundWithQuotes, batchedId)))
# runs insert into MS database for each batch of IDs
for idString in formattedIdStrings:
insertIntoMSDatabase(idString)
print("MSQL table loaded, Data inserted into destination")
endTime = time.time()
print("Program Time Elapsed: ",endTime-beginTime)
conn.close()
conn1.close()
mycursor.executemany(sql, val)
pyodbc.ProgrammingError: The second parameter to executemany must not be empty.
Before calling .executemany() you need to verify that val is not an empty list (as would be the case if .fetchall() is called on a SELECT statement that returns no rows) , e.g.,
if val:
mycursor.executemany(sql, val)
I am trying to find the time until the next day according to IST. I am using pytz. I am not getting why "tomorrow" is offset naive while tomorrow is defined using offset aware constants.
I've tried printing stuff like print tomorrow_date - now , but that gives 0 and i do not know how is that happening.
from datetime import datetime,timedelta
from pytz import timezone
ist = timezone('Asia/Kolkata')
def get_until_tomorrow():
now = datetime.now(ist)
#today = date.today()
tomorrow_date = now + timedelta(days=1)
tomorrow = datetime.combine(tomorrow_date,time=time(00,00))
seconds_left = tomorrow - now
return seconds_left.seconds
print(get_until_tomorrow())
I am getting the error that I cannot subtract an offset naive and an offset aware object. now is an offset aware object because i set it so directly, but tomorrow is, according to it, an offset naive variable. How is this possible when i have used only offset aware variables to define tomorrow?
Looks like you have to make it offset aware by using localise. See if this helps.
from datetime import datetime,timedelta, time
from pytz import timezone
ist = timezone('Asia/Kolkata')
def get_until_tomorrow():
now = datetime.now(ist)
tomorrow_date = now + timedelta(days=1)
tomorrow = datetime.combine(tomorrow_date,time=time(00,00))
tomorrow = ist.localize(tomorrow)
seconds_left = tomorrow - now
return seconds_left.seconds
print(get_until_tomorrow())
I get input JSON data from JS. This is a simple object, among which there is a date in the format "DD.MM.YYYY" - just a string.
If there is no dateStart in the object, I have to replace it with the current date (in withDefault).
paramsDecoder : Decode.Decoder Params
paramsDecoer =
Decode.succeed Params
|> Decode.andMap (Decode.field "dateStart" (Decode.string) |> (Decode.withDefault) "")
|> Decode.andMap (Decode.field "dateEnd" (Decode.string) |> (Decode.withDefault) "")
|> Decode.andMap (Decode.field "country" (Decode.string) |> (Decode.withDefault) "spain")
How can I do this in ELM?
Timezone is not important and is always equal to the same region.
I found an example of Time.now Time.zone usage,
but there time is getting in Update and its too late.
I solved this in two parts:
Part 1
Send the current time in to Elm when it initializes, using flags:
Elm.Main.init({flags: Date.now()});
And put it in your model:
import Time exposing (Posix, millisToPosix)
type alias Model = { now : Time.Posix, ... }
init : Int -> (Model, Cmd Msg)
init time = (Model (millisToPosix time), Cmd.none)
For your use case, you can then use withDefault model.now.
Part 2
The solution in Part 1 will only set now to the time when the page is loaded.
If you want to keep an up to date time you can use Time.every to update your model:
import Time exposing (Posix, millisToPosix, every)
timeOutCheck : Sub Msg
timeOutCheck = Time.every 250 UpdateTimeNow
type Msg = UpdateTimeNow Posix | ...
update msg model = case msg of
UpdateTimeNow time = { model | now = time }
...
This will ensure now is never more than 250 ms behind the current time. You can change 250 to suit your need.
I've been toying with Effects.tick and Time in one of my apps, and I can't seem to get my delays to be around the Time.second that it claims to be. It's nearly immediate. I understand that type alias Time = Float, and from my logs it seems that second = 1000, but this just burns so quickly, even with logging. Is there a clear explanation for this?
Effects.tick causes an action to be called nearly instantaneously, but the action that is being called gets passed a value of the current time of the tick. If you want to delay something by a second while using Effects.tick, you'll have to keep track of a starting point and compare it to the time of the current tick, and that's where you can add in Time.second.
Take this arbitrary example (you can paste it into http://elm-lang.org/try):
import Html exposing (..)
import Html.Events exposing (..)
import Html.Attributes exposing (..)
import StartApp
import Effects exposing (Never)
import Task
import Signal
import Time exposing (..)
app =
StartApp.start
{ init = init
, view = view
, update = update
, inputs = [ ]
}
main =
app.html
type alias Model =
{ lastTick : Maybe Time
, tickEverySecond : Maybe Time
}
init =
({ lastTick = Nothing, tickEverySecond = Nothing }, Effects.tick MyTicker)
view address model =
div []
[ div [] [ text <| "Current tick: " ++ (toString model.lastTick) ]
, div [] [ text <| "Updated every second: " ++ (toString model.tickEverySecond) ]
]
type Action
= MyTicker Time
update action model =
let
everySecond = Maybe.withDefault 0 model.tickEverySecond
getTickEverySecond time =
if time > everySecond + Time.second then
Just time
else
Just everySecond
in
case action of
MyTicker time -> (
{ model | lastTick = Just time
, tickEverySecond = getTickEverySecond time
}, Effects.tick MyTicker)
port tasks : Signal (Task.Task Never ())
port tasks =
app.tasks
Every update call requests a new tick, so it will just spin forever. The important part is that tickEverySecond is only updated if the last time it was updated is greater than one second ago.
You ask about the time precision. If you run that example, you'll notice that the increment every second is somewhat approximate; it will drift greater than a second. That isn't due to any kind of underlying imprecision. Remember that Elm's Time functionality is a thin veneer over Javascript's time and timer functionality. That drift is merely an artifact of all the tiny little delays caused by running code that responds to a timer signal.