I'm trying to create a full text search using . It works well in one table. With this, my database has a many-to-many relationship:
from flask_msearch import Search
search = Search()
app = Flask(__name__)
tasks_topics= db.Table('tasks_topics',
db.Column('task_id', db.Integer, db.ForeignKey('tasks.id')),
db.Column('topics_id', db.Integer, db.ForeignKey('topics.id'))
)
class Tasks(db.Model):
__tablename__ = 'tasks'
__searchable__ = ['task','id','topics.name'] # these fields will be indexed by whoosh
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
url = db.Column(db.String(120))
task = db.Column(db.Text)
topics = db.relationship('Topics',
secondary=tasks_topics,
back_populates="tasks")
class Topics(db.Model):
__tablename__ = 'topics'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(140))
parent = db.Column(db.Integer, default=0)
# связи
tasks = db.relationship('Tasks',
secondary=tasks_topics,
back_populates="topics")
search.create_index(Tasks)
results = Tasks.query.msearch('Find x',fields=['task']).all()
print(results)
In documentation flask-msearch example shows one go many relationship. How should I solve this problem?
I am trying to get itemwise inventory desired result is like this
[
{
'item' : shoes,
'total_unit' : 134
},
{
'item': sneaker,
'unit': 100
}
]
but I am getting result like
[
{
"item": 5,
"unit": 134
},
{
"item": 4,
"unit": 100
}
]
I want to get item name instead of id
Models.py
class Item(models.Model):
item_name = models.CharField(max_length=50)
srn_code = models.CharField(max_length=20,unique=True)
category = models.ForeignKey(Category,related_name='categories',on_delete=models.CASCADE)
def __str__(self):
return self.item_name
class Transaction(models.Model):
category = models.ForeignKey(Category,on_delete=models.CASCADE)
item = models.ForeignKey(Item,on_delete=models.CASCADE)
size = models.CharField(blank=True,max_length=20)
unit = models.IntegerField()
unit_price = models.DecimalField(decimal_places=2,max_digits=20)
supplier = models.ForeignKey(Supplier,on_delete=models.CASCADE)
tran_date = models.DateField(auto_now=False,auto_created=False,blank=False)
created_date = models.DateTimeField(auto_now=True)
def __str__(self):
return f"{self.item.item_name} {self.unit}"
serializer.py
class InventorySerializer(serializers.ModelSerializer):
item = serializers.CharField(read_only=True)
total_unit = serializers.IntegerField(read_only=True)
class Meta():
model = Transaction
fields = ['item','total_unit']
views.py
class InventoryModelViewSet(ModelViewSet):
queryset = Transaction.objects.all()
serializer_class = InventorySerializer
def get_queryset(self):
return Transaction.objects.values('item').annotate(
total_unit = Sum('unit')
).order_by('item')
Thanks for posting my question, I have sorted out my question, my solution is below, if anybody needs or gets query like this, then its might be helpful.
I have declared a method in serializers class for get item_name from another table
class InventorySerializer(serializers.ModelSerializer):
item = serializers.CharField(read_only=True)
total_unit = serializers.IntegerField(read_only=True)
item_name = serializers.SerializerMethodField()
class Meta():
model = Transaction
fields = ['item','item_name','total_unit']
def get_item_name(self, obj):
item_obj = Item.objects.filter(id=obj['item']).first()
item_name = model_to_dict(item_obj)
return item_name['item_name']
`
I am implementing a simple blog in wagtail. In the case of the blog pages the search should also look into the two custom fields 'intro' and 'body'. My BlogPage-model is as follows:
class PageWithSidebar(Page):
def get_context(self, request):
context = super(PageWithSidebar, self).get_context(request)
context['tags'] = BlogPageTag.objects.all().select_related().values('tag_id', 'tag_id__name').annotate(item_count=Count('tag_id')).order_by('-item_count')[:10]
context['categories'] = BlogCategory.objects.values('name').annotate(Count('name')).values('name').order_by('name')
context['recent_blogpages'] = Page.objects.filter(content_type__model='blogpage').filter(live='1').order_by('-first_published_at')
return context
class BlogPage(PageWithSidebar):
date = models.DateField("Post date")
intro = models.CharField(max_length=250)
body = RichTextField(blank=True)
tags = ClusterTaggableManager(through=BlogPageTag, blank=True)
categories = ParentalManyToManyField('blog.BlogCategory', blank=True)
social_description = models.CharField(max_length=140, blank=True)
def main_image(self):
gallery_item = self.gallery_images.first()
if gallery_item:
return gallery_item.image
else:
return None
def main_image_caption(self):
gallery_item = self.gallery_images.first()
if gallery_item:
return gallery_item.caption
else:
return None
search_fields = PageWithSidebar.search_fields + [
index.SearchField('intro'),
index.SearchField('body'),
]
content_panels = PageWithSidebar.content_panels + [
MultiFieldPanel([
FieldPanel('date'),
FieldPanel('tags'),
FieldPanel('categories', widget=forms.CheckboxSelectMultiple),
FieldPanel('social_description'),
], heading="Blog information"),
FieldPanel('intro'),
FieldPanel('body'),
InlinePanel('gallery_images', label="Gallery images"),
]
The search works just fine for the 'title' field but not for the two custom fields. I just get no results if I search for words which are just contained in the 'intro' or 'body' field.
Any ideas what i am missing?
I didn't know that the default search backend doesn't support custom fields. After switching to elasticsearch the custom fields were included.
I have some models:
class Paper(db.Model):
# Regarding lack of unique constraints -- the legacy database had
# much duplicate data, and I hope to eventually eliminate that duplicate
# data and then add uniqueness constraints on some columns later.
__tablename__ = 'papers'
__searchable__ = ['title', 'abstract', 'keywords']
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(500))
abstract = db.Column(db.Text, nullable=True)
doi = db.Column(db.String(50), nullable=True)
pubmed_id = db.Column(db.String(50), nullable=True)
link = db.Column(db.String(500), nullable=True)
journals = db.relationship(Journal, secondary="journal_paper")
chapters = db.relationship(Chapter, secondary="chapter_paper")
authors = db.relationship(Author, secondary="author_paper")
keywords = db.relationship(Keyword, secondary="keyword_paper")
...
I have a custom model view:
class PaperModelView(MainModelView): # Looked up by advanced search with check if __repr__ is "PaperModelView object"
page_size = 100
column_list = (
'title',
'chapter_paper_assoc',
)
column_exclude_list = (
'keyword_paper_assoc',
'author_paper_assoc',
'link',
'doi',
'pubmed_id',
)
column_details_list = (
'title',
'chapter_paper_assoc',
'journal_paper_assoc',
'authors',
'abstract',
'keywords',
'doi',
'pubmed_id',
'link',
)
column_labels = {
'chapter_paper_assoc': 'Print Information',
'journal_paper_assoc': 'Publication Information',
}
column_filters = [
'chapter_paper_assoc.printed',
'journal_paper_assoc.publication_date',
'chapters.name',
'chapters.number',
'journals.name',
'authors.first_name',
'authors.last_name',
'keywords.keyword',
'doi',
'abstract',
'link',
'title',
]
column_filter_labels = {
'chapter_paper_assoc.printed': 'Printed',
'journal_paper_assoc.publication_date': 'Publication Date',
'chapters.name': 'Chapter Name',
'chapters.number': 'Chapter Number',
'journals.name': 'Journal Name',
'authors.first_name': 'Author First Name',
'authors.last_name': 'Author Last Name',
'keywords.keyword': 'Keyword',
'doi': 'DOI',
'abstract': 'Abstract',
'link': 'Link',
'title': 'Paper Title',
}
column_details_exclude_list = (
'keyword_paper_assoc',
'author_paper_assoc',
)
form_excluded_columns = (
'chapter_paper_assoc',
'journal_paper_assoc',
'keyword_paper_assoc',
'author_paper_assoc',
)
column_formatters = {
'journals': macro('render_journals'),
'chapters': macro('render_chapters'),
'authors': macro('render_authors'),
'keywords': macro('render_keywords'),
'chapter_paper_assoc': macro('render_chapter_papers'),
'journal_paper_assoc': macro('render_journal_papers'),
}
column_formatters_export = {
'journals': formatter,
'chapters': formatter,
'authors': formatter,
'keywords': formatter,
'chapter_paper_assoc': formatter,
'journal_paper_assoc': formatter,
}
column_searchable_list = (
'title',
'keywords.keyword',
)
form_ajax_refs = {
'authors': {
'fields': ['first_name', 'middle_name', 'last_name'],
'page_size': 10,
},
'chapters': {
'fields': ['number', 'name',],
'page_size': 5,
},
'journals': {
'fields': ['name', 'issn'],
'page_size': 2,
},
'keywords': {
'fields': ['keyword',],
'page_size': 10,
}
}
def unique_title(form, field):
p = Paper.query.filter_by(title=field.data).first()
if p:
raise ValidationError("A Paper with this title already exists")
def unique_doi(form, field):
p = Paper.query.filter_by(doi=field.data).first()
if p:
raise ValidationError("A Paper with this doi already exists")
form_args = {
'title': {
'validators': [unique_title, DataRequired()]
},
'doi': {
'validators': [unique_doi],
},
'link': {
'validators': [DataRequired(), URL()],
},
'journals': {
'validators': [Length(0,1)], # Thought it is a many-many field, allow only 0 or 1
} # 0 or 1 Journals assumption is carried to on_model_change (be careful if changing)
}
form_base_class = FlaskForm
list_template = 'auth/model/paper/list.html'
def on_model_change(self, form, model, is_created):
"""
Perform some actions before a model is created or updated.
Called from create_model and update_model in the same transaction (if it has any meaning for a store backend).
By default does nothing.
Parameters:
form – Form used to create/update model
model – Model that will be created/updated
is_created – Will be set to True if model was created and to False if edited
"""
all_chapters = list(set(form.chapters.data + form.chapters_printed.data))
for chapter in all_chapters:
if chapter in form.chapters_printed.data: # if chapter in both, printed takes priority
chapter_paper = ChapterPaper.query.filter_by(chapter_id=chapter.id, paper_id=model.id).first()
if not chapter_paper:
chapter_paper = ChapterPaper(chapter_id=chapter.id, paper_id=model.id)
chapter_paper.printed = True
db.session.add(chapter_paper)
journal = None
if form.journals.data:
journal = form.journals.data[0]
if journal: # Assumes only 1 journal if there are any journals in this field
issue = form.issue.data
volume = form.volume.data
pages = form.pages.data
journal_paper = JournalPaper.query.filter_by(journal_id=journal.id, paper_id=model.id).first()
if not journal_paper:
journal_paper = JournalPaper(journal_id=journal.id, paper_id=model.id)
journal_paper.issue = issue
journal_paper.volume = volume
journal_paper.pages = pages
db.session.add(journal_paper)
db.session.commit()
#expose('/new/', methods=('GET', 'POST'))
def create_view(self):
"""
Create model view
"""
return_url = get_redirect_target() or self.get_url('.index_view')
if not self.can_create:
return redirect(return_url)
form = ExtendedPaperForm()
if not hasattr(form, '_validated_ruleset') or not form._validated_ruleset:
self._validate_form_instance(ruleset=self._form_create_rules, form=form)
if self.validate_form(form):
# in versions 1.1.0 and before, this returns a boolean
# in later versions, this is the model itself
model = self.create_model(form)
if model:
flash('Record was successfully created', 'success')
if '_add_another' in request.form:
return redirect(request.url)
elif '_continue_editing' in request.form:
# if we have a valid model, try to go to the edit view
if model is not True:
url = self.get_url('.edit_view', id=self.get_pk_value(model), url=return_url)
else:
url = return_url
return redirect(url)
else:
# save button
return redirect(self.get_save_return_url(model, is_created=True))
form_opts = FormOpts(widget_args=self.form_widget_args,
form_rules=self._form_create_rules)
if self.create_modal and request.args.get('modal'):
template = self.create_modal_template
else:
template = self.create_template
return self.render(template,
form=form,
form_opts=form_opts,
return_url=return_url)
def on_form_prefill(self, form, id):
"""
Perform additional actions to pre-fill the edit form.
Called from edit_view, if the current action is rendering
the form rather than receiving client side input, after
default pre-filling has been performed.
By default does nothing.
You only need to override this if you have added custom
fields that depend on the database contents in a way that
Flask-admin can't figure out by itself. Fields that were
added by name of a normal column or relationship should
work out of the box.
:param form:
Form instance
:param id:
id of the object that is going to be edited
"""
model = Paper.query.filter_by(id=id).first()
form.title.data = model.title
form.abstract.data = model.abstract
form.pubmed_id.data = model.pubmed_id
form.doi.data = model.doi
form.link.data = model.link
form.chapters.data = model.chapters
form.authors.data = model.authors
form.keywords.data = model.keywords
form.chapters_printed.data = model.get_printed_chapters()
form.journals.data = model.journals
journal = model.journals[0]
if journal:
journal_paper = JournalPaper.query.filter_by(paper_id=model.id, journal_id=journal.id).first()
if journal_paper:
form.issue.data = journal_paper.issue
form.volume.data = journal_paper.volume
form.pages.data = journal_paper.pages
form.publication_date = journal_paper.publication_date
#expose('/edit/', methods=('GET', 'POST'))
def edit_view(self):
"""
Edit model view
"""
return_url = get_redirect_target() or self.get_url('.index_view')
if not self.can_edit:
return redirect(return_url)
id = get_mdict_item_or_list(request.args, 'id')
if id is None:
return redirect(return_url)
model = self.get_one(id)
if model is None:
flash('Record does not exist!', 'error')
return redirect(return_url)
form = ExtendedPaperForm()
if not hasattr(form, '_validated_ruleset') or not form._validated_ruleset:
self._validate_form_instance(ruleset=self._form_edit_rules, form=form)
if self.validate_form(form):
if self.update_model(form, model):
flash('Record was successfully saved', 'success')
if '_add_another' in request.form:
return redirect(self.get_url('.create_view', url=return_url))
elif '_continue_editing' in request.form:
return redirect(request.url)
else:
# save button
return redirect(self.get_save_return_url(model, is_created=False))
if request.method == 'GET':
self.on_form_prefill(form, id)
form_opts = FormOpts(widget_args=self.form_widget_args,
form_rules=self._form_edit_rules)
if self.edit_modal and request.args.get('modal'):
template = self.edit_modal_template
else:
template = self.edit_template
return self.render(template,
model=model,
form=form,
form_opts=form_opts,
return_url=return_url)
#property
def can_create(self):
return current_user.can_edit_papers()
#property
def can_edit(self):
return current_user.can_edit_papers()
#property
def can_delete(self):
return current_user.can_edit_papers()
def is_accessible(self):
if current_user.is_authenticated:
return current_user.can_view_papers()
return False
# Add a batch action to this class using flask admin's syntax
#action('cite', 'Cite', 'Create Citations for the Selected Papers?')
def action_cite(self, ids):
try:
query = Paper.query.filter(Paper.id.in_(ids))
citations = ""
for paper in query.all():
citation = paper.cite('APA')
citations += "{}\n".format(citation)
response = make_response(citations)
response.headers[
"Content-Disposition"
] = "attachment; filename=citations.txt" # Downloadable response
return response
except Exception as ex:
if not self.handle_view_exception(ex):
raise
#action('chapter_add', 'Add to Chapters')
def action_chapter_add(self, ids):
return_url = request.referrer
try:
return redirect(
url_for('addtochapterview.index',
return_url=return_url), 307
)
except Exception as ex:
if not self.handle_view_exception(ex):
raise
#action('chapter_remove', 'Remove From Chapters')
def action_chapter_remove(self, ids):
return_url = request.referrer
try:
return redirect(
url_for('removefromchapterview.index',
return_url=return_url), 307
)
except Exception as ex:
if not self.handle_view_exception(ex):
raise
#action('mark_as_printed', 'Mark As Printed')
def mark_printed(self, ids):
return_url = request.referrer
try:
return redirect(
url_for('markasprintedview.index',
return_url=return_url), 307
)
except Exception as ex:
if not self.handle_view_exception(ex):
raise
#action('mark_as_unprinted', 'Mark As NOT Printed')
def unmark_printed(self, ids):
return_url = request.referrer
try:
return redirect(
url_for('markasunprintedview.index',
return_url=return_url), 307
)
except Exception as ex:
if not self.handle_view_exception(ex):
raise
I have a custom form:
class ExtendedPaperForm(FlaskForm):
title = StringField()
abstract = TextAreaField()
doi = StringField()
pubmed_id = StringField()
link = StringField()
journals = QuerySelectMultipleField(
query_factory=_get_model(Journal),
allow_blank=False,
)
issue = StringField()
volume = StringField()
pages = StringField()
publication_date = DateField(format='%Y-%m-%d')
authors = QuerySelectMultipleField(
query_factory=_get_model(Author),
allow_blank=False,
)
keywords = QuerySelectMultipleField(
query_factory=_get_model(Keyword),
allow_blank=True,
)
chapters_printed = QuerySelectMultipleField(
query_factory=_get_model(Chapter),
allow_blank=True,
label="Chapters (Printed)",
)
chapters = QuerySelectMultipleField(
query_factory=_get_model(Chapter),
allow_blank=True,
label="Chapters (All)",
)
with the custom query factory:
def _get_model(model, order=None):
if order:
return lambda: db.session.query(model).order_by(order)
return lambda: db.session.query(model)
This all works very well, with the exception that it is very slow. From the time that the user clicks the "Create" button to the time that the servers returns a response is, on average, around ~1.1 minutes. That's too long :/.
So, I think my options are:
1) Write a better query factory function
2) Memoize the query factories for each field of the form
3) Some combination of the two
I don't know how to do any of these options. How would one even write a better query factory here. I am pretty sure that there isn't much to do on this end, since I need every option to be available for each field. I have used Flask-Cache before on a simpler project, but don't know how to apply this knowledge here.
I imagine that I could simply memoize the query functions -- but I don't know where in my model view I would clear the cached values and rebuild them -- how often -- why -- etc.
I don't have the strongest web programming vocabulary, so using references and documentation is proving frustrating. Could someone please provide some guidance on how to improve this process? Or like, maybe just the right place to look for inspiration in the flask-admin source code?
P.S
If you have a general guide for scaling and optimizing a flask app that you think would help, please share in comments. Thanks.
No help here, and yes, sorry that this post was so vague.
Here are some things that I did that reduced the average client response time from 95,000 ms to the expected / acceptable ~100 ms:
Integrated Caching queries
Eliminated joins (used dynamic loading of related objects where possible)
Cached query factories and update them on model change
The Flask-DebugToolbar extension is great! You should use it.
I want to create a supplier model i.e code, name, address, phone
I am also going to use the aurelia-validation classes.
Using them both together with the viewmodel - SupplierMaint.js & view - SupplierMaint.html is fine but I want to keep the Supplier model in another file so that I can use it in many places to ensure a consistent validation of values lengths etc.
How should I go about this?
I suspect the #NoView annotation might have something to do with it but I am not sure how to import the external file and wire it up to the submit & validation processing in the SupplierMaint.js
Bob
Models
This is how I do it -
export class Supplier {
name = '';
modelId = '';
constructor(data) {
Object.assign(this, data);
}
}
This let's anyone consume your model and create a new instance of it. The data that is passed in will override the default values you define, but if they do not you still have those default values.
Validation
Validation is currently in a state of flux. I'm hoping to be done with the initial refactoring work by the end of this weekend but here is a preview for what I propose as the best usage, feedback welcome.
import {ValidationReporter} from 'aurelia-validate';
export class PersonViewModel {
activePerson;
static inject = [ValidationReporter];
constructor(reporter) {
this.activePerson = new PersonModel({firstName: 'Lucky Luke'});
this.reporter = reporter.subscribe(validationErrors => {
// do something with validationErrors
});
}
}
class PersonModel {
#length({ minimum: 5, maximum: 25 }) firstName = 'Luke';
#required lastName = 'Skywalker';
#date lastUpdated = new Date();
#datetime lastTimeUpdated = new Date();
#email email = 'luke#skywalker.net';
#length({ minimum: 5, maximum: 25 }) password = 'equal';
#equality confirmPassword = 'equal';
#url website = 'http://www.google.com';
#numericality friendCount = 25;
#numericality({ noStrings: true }) age = 25;
constructor(data) {
Object.assign(this, data);
}
}