My goal is to transform the elasticsearch result to an rxjs stream and thought of doing so using the scroll API fetching 1 data point on every call. However it seems that my rxjs stream returns no results for the second elastic query (searchElastic).
Below is a sample of my code:
import * as Rx from 'rxjs';
import {elasticClient} from '../Helpers.ts';
function searchElastic({query, sort}) {
const body: any = {
size: 1,
query,
_source: { excludes: ['logbookType', 'editable', 'availabilityTag'] },
sort
};
// keep the search results "scrollable" for 30 secs
const scroll = '30s';
return Rx.Observable
.fromPromise(elasticClient.search({ index: 'data', body, scroll }))
.concatMap(({_scroll_id, hits: {hits}}) => {
const subject = new Rx.Subject();
if(hits.length) {
// initial data
subject.onNext(hits[0]._source as ElasticDoc);
console.log(hits[0]._id);
const handleDoc = (err, res) => {
if(err) {
subject.onError(err);
return;
}
const {_scroll_id, hits: {hits}} = res;
if(!hits.length) {
subject.onCompleted();
} else {
subject.onNext(hits[0]._source as ElasticDoc);
console.log(hits[0]._id);
setImmediate(() =>
elasticClient.scroll({scroll, scrollId: _scroll_id},
handleDoc));
}
};
setImmediate(() =>
elasticClient.scroll({scroll, scrollId: _scroll_id},
handleDoc));
} else {
subject.onCompleted();
}
return subject.asObservable();
});
}
function getEntries() {
const entriesQuery = {
query: {
filtered: {
filter: {
bool: {
must: [{
range: {
creationTimestamp: {
gte: "2018-04-01T07:55:59.915Z",
lte: "2018-04-01T07:57:08.915Z"
}
}
}, {
query: {
query_string: {
query: "+type:*scan*"
}
}
}]
}
}
}
},
sort: [{
creationTimestamp: {
order: "asc"
},
id: {
order: "asc"
}
}]
};
return searchElastic(entriesQuery)
.concatMap(entry => {
// all entries are logged correctly
console.log(entry.id);
// array that contains MongoDB _ids as strings
const ancestors = entry.ancestors || [];
// if no parents => doesn't match
if(!ancestors.length) {
return Rx.Observable.empty();
}
const parentsQuery = {
query: {
filtered: {
filter: {
bool: {
must: [{
range: {
creationTimestamp: {
gte: "2018-04-01T07:55:59.915Z",
lte: "2018-04-01T07:57:08.915Z"
}
}
}, {
query: {
query_string: {
query: "+type:*block* +name:*Report*"
}
}
}]
}
}
}
},
sort: [{
creationTimestamp: {
order: "asc"
},
id: {
order: "asc"
}
}]
};
parentsQuery.query.filtered.filter.bool.must.push({
terms: {
id: ancestors
}
});
// fetch parent entries
return searchElastic(parentsQuery)
.count()
.concatMap(count => {
// count is always 0 even though entries are logged
// in the searchElastic console.logs
console.log(entry.id, count);
return Rx.Observable.just(entry);
});
});
}
function executeQuery() {
try {
getEntries()
.subscribe(
(x) => console.log(x.id),
err => console.error(err),
() => {}
)
} catch(e) {
console.error(e);
}
}
Looks like it's an rxjs problem, since all ancestors entries get logged but count always returns 0.
P.S. using elasticsearch v1.7
After playing with a couple rxjs examples with subjects, it seems like the subject is being completed (onCompleted) before an observer subscribes to it.
Working example
var subject = new Rx.Subject();
var subscription = subject.subscribe(
function(x) {
console.log('onNext: ' + x);
},
function(e) {
console.log('onError: ' + e.message);
},
function() {
console.log('onCompleted');
});
subject.onNext(1);
// => onNext: 1
subject.onNext(2);
// => onNext: 2
subject.onCompleted();
// => onCompleted
<!DOCTYPE html>
<html>
<head>
<script src="//code.jquery.com/jquery-2.1.0.min.js"></script>
<title>JS Bin</title>
<script src="//cdn.jsdelivr.net/rsvp/3.0.6/rsvp.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/rxjs/2.2.28/rx.all.min.js"></script>
</head>
</html>
Broken example
var subject = new Rx.Subject();
subject.onNext(1);
// => onNext: 1
subject.onNext(2);
// => onNext: 2
subject.onCompleted();
// => onCompleted
var subscription = subject.subscribe(
function(x) {
console.log('onNext: ' + x);
},
function(e) {
console.log('onError: ' + e.message);
},
function() {
console.log('onCompleted');
});
<!DOCTYPE html>
<html>
<head>
<script src="//code.jquery.com/jquery-2.1.0.min.js"></script>
<title>JS Bin</title>
<script src="//cdn.jsdelivr.net/rsvp/3.0.6/rsvp.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/rxjs/2.2.28/rx.all.min.js"></script>
</head>
</html>
So I fixed it by changing searchElastic to the following:
function searchElasticStream({query, sort}) {
const body: any = {
size: 1,
query,
_source: { excludes: ['logbookType', 'editable', 'availabilityTag'] },
sort
};
// keep the search results "scrollable" for 30 secs
const scroll = '30s';
return Rx.Observable
.fromPromise(elasticClient.search({ index: 'data', body, scroll }))
.flatMap(({_scroll_id, hits: {hits}}) => {
const subject = new Rx.Subject();
// this made the difference
setImmediate(() => {
if(hits.length) {
// initial data
subject.onNext(hits[0]._source as ElasticDoc);
const handleDoc = (err, res) => {
if(err) {
subject.onError(err);
return;
}
const {_scroll_id, hits: {hits}} = res;
if(!hits.length) {
subject.onCompleted();
} else {
subject.onNext(hits[0]._source as ElasticDoc);
setImmediate(() =>
elasticClient.scroll({scroll, scrollId: _scroll_id},
handleDoc));
}
};
setImmediate(() =>
elasticClient.scroll({scroll, scrollId: _scroll_id},
handleDoc));
} else {
subject.onCompleted();
}
});
return subject.asObservable();
});
}
Related
I'm using Strapi 4, and I try to add computed field to my custom resolver. (I'm not a graphql expert). I've followed this tutorial to do it.
https://www.theitsolutions.io/blog/how-to-add-custom-graphql-query-to-strapi-v4
I’m also using the “toEntityResponseCollection” methods to send the datas and display it in graphql playground.
But, when I send it back, I get a null result.
Here is my custom resolver
"use strict";
module.exports =
(strapi, toEntityResponseCollection, toEntityResponse) =>
({ nexus }) => ({
typeDefs: `
type PopularityResponse {
id: ImpressionEntityResponseCollection
startDate: String
endDate: String
branding: String
}
extend type Query {
popularity(id: ID!, startDate: String, endDate: String, branding: String): PopularityResponse
}
`,
resolvers: {
Query: {
popularity: {
resolve: async (parent, args, context) => ({
id: args.id,
startDate: args.startDate,
endDate: args.endDate,
branding: args.branding,
}),
},
},
PopularityResponse: {
id: {
resolve: async (parent, args) => {
let query = {
value: await strapi.entityService.findMany(
"api::impression.impression",
{
filters: {
googleid: {
id: {
$eq: parent.id,
},
},
date_debut: {
$gte: parent.startDate,
},
date_fin: {
$lte: parent.endDate,
},
},
},
args
),
};
console.log(query.value);
console.log(parent);
let aggregate = query.value.reduce(
(acc, key) => {
// vérifie si la campagne est dans la liste
if (
[parent.branding].some((elem) => {
let reg = new RegExp(elem);
return reg.test(key.campaignName);
})
) {
let brandingIndex = acc.branding.findIndex(
(el) => el.date_debut == key.date_debut
);
if (brandingIndex !== -1) {
// si on a un élément
acc.branding[brandingIndex].search_impression_share +=
parseInt(key.search_impression_share);
} else {
acc.branding.push({
search_impression_share: parseInt(
key.search_impression_share
),
date_debut: key.date_debut,
});
}
} else {
let nobrandingIndex = acc.nobranding.findIndex(
(el) => el.date_debut == key.date_debut
);
if (nobrandingIndex !== -1) {
// si on a un élément
acc.nobranding[nobrandingIndex].search_impression_share +=
parseInt(key.search_impression_share);
} else {
acc.nobranding.push({
search_impression_share: parseInt(
key.search_impression_share
),
date_debut: key.date_debut,
});
}
}
return acc;
},
{ branding: [], nobranding: [] }
);
console.log("==========>>>>",aggregate);
let y = [query.value[0]];
return toEntityResponseCollection([aggregate]);
},
},
},
},
resolversConfig: {
"Query.popularity": {
auth: {
scope: [
"api::impression.impression.findOne",
"api::impression.impression.find",
],
},
},
},
});
Here is my graphql query
query GetPopularity {
popularity(id: "37", startDate:"2022-06-13",endDate:"2022-07-15",branding:"brand") {
myData {
data {
attributes {
googleid {
data {
attributes {
g_customer_id
}
}
}
}
}
}
}
}
When I log the result ssr, I get my computed datas, but when I look at grapql Playground, I get null.
{
"data": {
"popularity": {
"id": {
"data": [
{
"attributes": {
"search_impression_share": null,
"search_top_impression_share": null
}
}
]
}
}
}
}
I don't know what to do to make it work.
I do it like this, because I need to fetch a huge amount of datas. I know that strapi has a 100 limit result from graphql. Even if I can manualy increase it in the config file, I understand it's not a good practice.
If you have any idea how to solve this, please let me know.
Thanks
Fabien
I found how to solve my issue.
I’ve created a specific type which aggregate the datas.
Now I’m able to fetch my computed elements.
"use strict";
module.exports =
(strapi, toEntityResponseCollection, toEntityResponse) =>
({ nexus }) => ({
typeDefs: `
type PopularityResponse {
id: ImpressionEntityResponseCollection
startDate: String
endDate: String
branding: String
aggregated: aggregateInput
}
type aggregateInput {
brand: [singleAggregate]
nobrand: [singleAggregate]
}
type singleAggregate {
date_debut: String
search_impression_share: Int
}
extend type Query {
popularity(id: ID!, startDate: String, endDate: String, branding: String): PopularityResponse
}
`,
resolvers: {
Query: {
popularity: {
resolve: async (parent, args, context) => ({
id: args.id,
startDate: args.startDate,
endDate: args.endDate,
branding: args.branding,
}),
},
},
PopularityResponse: {
aggregated: {
resolve: async (parent, args, ctx) => {
let compile = await strapi.entityService.findMany(
"api::impression.impression",
{
filters: {
googleid: {
id: {
$eq: parent.id,
},
},
date_debut: {
$gte: parent.startDate,
},
date_fin: {
$lte: parent.endDate,
},
},
},
args
);
// console.log(compile);
let aggregate = compile.reduce(
(acc, key) => {
// vérifie si la campagne est dans la liste
if (
[parent.branding].some((elem) => {
let reg = new RegExp(elem);
return reg.test(key.campaignName);
})
) {
let brandingIndex = acc.branding.findIndex(
(el) => el.date_debut == key.date_debut
);
if (brandingIndex !== -1) {
// si on a un élément
acc.branding[brandingIndex].search_impression_share +=
parseInt(key.search_impression_share);
} else {
acc.branding.push({
search_impression_share: parseInt(
key.search_impression_share
),
date_debut: key.date_debut,
});
}
} else {
let nobrandingIndex = acc.nobranding.findIndex(
(el) => el.date_debut == key.date_debut
);
if (nobrandingIndex !== -1) {
// si on a un élément
acc.nobranding[nobrandingIndex].search_impression_share +=
parseInt(key.search_impression_share);
} else {
acc.nobranding.push({
search_impression_share: parseInt(
key.search_impression_share
),
date_debut: key.date_debut,
});
}
}
return acc;
},
{ branding: [], nobranding: [] }
);
return {
brand: aggregate.branding,
nobrand: () => {
return aggregate.nobranding;
},
};
},
},
},
},
resolversConfig: {
"Query.popularity": {
auth: {
scope: [
"api::impression.impression.findOne",
"api::impression.impression.find",
],
},
},
},
});
````
I'd love to implement nested pagination within my application. I have been reading the docs and looking at several other examples but I just can't get this to work - any help is appreciated! Thanks!
React component:
I am clicking the button to run the fetchMore function provided by the useQuery hook (apollo). The network request is going through and the new products are merged into the cache... but no new products render on the page.
export const FilterableKit = () => {
const selectedKitId = useReactiveVar(selectedKitIdVar);
const [
getKitProducts,
{ data: getKitProductsData, loading: getKitProductsLoading, fetchMore },
] = useGetKitProductsLazyQuery();
useEffect(() => {
if (selectedKitId) {
getKitProducts({
variables: {
getKitsInput: {
_id: {
string: selectedKitId,
filterBy: "OBJECTID" as StringFilterByEnum,
},
},
getProductsInput: {
config: {
pagination: {
reverse: true,
limit: 3,
},
},
},
},
});
}
}, [getKitProducts, selectedKitId]);
const kitProducts = getKitProductsData?.getKits.data?.find(
(kit) => kit?._id === selectedKitId
)?.products.data;
const handleLoadMore = () => {
if (kitProducts && kitProducts?.length > 0) {
const remaining =
getKitProductsData?.getKits.data[0]?.products.stats?.remaining;
if (remaining && remaining > 0) {
const cursor =
kitProducts[kitProducts.length - 1] &&
kitProducts[kitProducts.length - 1]?.createdAt;
fetchMore({
variables: {
getProductsInput: {
config: {
pagination: {
reverse: true,
createdAt: cursor,
},
},
},
},
});
}
}
};
return (
<CContainer>
<KitItemCards products={kitProducts} loading={getKitProductsLoading} />
<CContainer className="d-flex justify-content-center my-3">
<CButton color="primary" className="w-100" onClick={handleLoadMore}>
Load More
</CButton>
</CContainer>
</CContainer>
);
};
Type Policies: I define the "Kit" typePolicy to merge products into the correct field.
export const cache: InMemoryCache = new InMemoryCache({
typePolicies: {
Kit: {
fields: {
products: {
keyArgs: false,
merge(existing = [] as Product[], incoming: GetProductsResponse) {
if (!incoming) return existing;
if (!existing) return incoming;
const { data: products, ...rest } = incoming;
let result: any = rest;
result = [...existing, ...(products ?? [])];
return result;
},
},
},
},
});
Thanks for any pointers in the right direction! Let me know if there is something else you'd like to see.
Hi Guys I'm trying to filter post with data json format field?
"categoryList": ["cat", "cat1"]
For anyone still looking for a solution, this is what I have done for a json type field called tags of a collection type called Articles.
I have two articles in the database with one article having the following values set:
title: "lorem ipsum 1",
tags: [
"test",
"rest"
]
The other article has the following values set:
title: "lorem ipsum 2",
tags: [
"test",
"graphql"
]
My graphql query looks like this:
query {
articlesByTag(limit: 2, where: {tags_include: ["test", "rest"]}, start: 0, sort: "title:asc") {
title,
tags
}
}
While my rest query looks like this:
http://localhost:1337/articlesByTag?limit=2&tags_include[]=test&tags_include[]=rest
This is my articles.js service file:
const { convertRestQueryParams, buildQuery } = require('strapi-utils');
const _ = require('lodash');
const { convertToParams, convertToQuery } = require('../../../node_modules/strapi-plugin-graphql/services/utils');
module.exports = {
async findByTag(ctx) {
let tags_include;
if (ctx.where && ctx.where.tags_include && ctx.where.tags_include.length > 0) {
tags_include = ctx.where.tags_include;
delete ctx.where.tags_include;
} else if (ctx.query && ctx.query.tags_include && ctx.query.tags_include.length > 0) {
tags_include = ctx.query.tags_include;
delete ctx.query.tags_include;
}
if (!Array.isArray(tags_include)) {
tags_include = [tags_include];
}
let filters = null;
if (ctx.query) {
filters = convertRestQueryParams({
...convertToParams(ctx.query)
});
} else {
filters = convertRestQueryParams({
...convertToParams(_.pick(ctx, ['limit', 'start', 'sort'])),
...convertToQuery(ctx.where),
});
}
const entities = await strapi.query('articles').model.query(qb => {
buildQuery({ model: strapi.query('articles').model, filters: filters })(qb);
if (tags_include.length > 0) {
tags_include.forEach((tag) => {
if (tag && tag.length > 0) {
const likeStr = `%"${tag}"%`;
qb.andWhere('tags', 'like', likeStr);
}
});
}
}).fetchAll();
return entities;
},
};
This is the entry needed in routes.js
{
"method": "GET",
"path": "/articlesByTag",
"handler": "articles.findByTag",
"config": {
"policies": []
}
}
This is the controller articles.js
const { sanitizeEntity } = require('strapi-utils');
module.exports = {
async findByTag(ctx) {
const entities = await strapi.services.articles.findByTag(ctx);
return entities.map(entity => sanitizeEntity(entity, { model: strapi.models.articles }));
},
};
And finally this is the schema.graphql.js
module.exports = {
query: `
articlesByTag(sort: String, limit: Int, start: Int, where: JSON): [Articles]
`,
resolver: {
Query: {
articlesByTag: {
description: 'Return articles filtered by tag',
resolverOf: 'application::articles.articles.findByTag',
resolver: async (obj, options, ctx) => {
return await strapi.api.articles.controllers.articles.findByTag(options);
},
},
},
},
};
There is not currently a way to filter the JSON fields yet as of beta.17.8 (latest)
Probably something like that?
strapi.query('cool_model').find({ categoryList: { $all: [ "cat" , "cat1" ] } })
I've a situation like this...
const INITIAL_STATE = {
chat: []
}
Then I set the chat and I include this data:
[
{
"otherParty":"aaaaa",
"thread":[
{
"a":1,
"b":2,
"c":3
},
{
"d":4,
"e":5,
"f":6
}
]
},
{
"otherParty":"bbbb",
"thread":[
{
"a":1,
"b":2,
"c":3
},
{
"d":4,
"e":5,
"f":6
}
]
},
{
"otherParty":"cccc",
"thread":[
{
"a":1,
"b":2,
"c":3
},
{
"d":4,
"e":5,
"f":6
}
]
}
]
I need to add a new item at array[1].thread something like { g: 7, h: 8, i: 9 } - In other words: I'd like to specify the index of the array and add a new thread.
How to archive this ?
export const addNewThread = (obj, index) => {
return {
type: ADD_NEW_THREAD,
payload: {
thread: obj,
index: index
}
}
}
and the reducer...(I need to fill the ????)
const INITIAL_STATE = {
chat: []
}
export default (state = INITIAL_STATE, action) => {
switch (action.type) {
case ADD_NEW_THREAD:
return {
...state,
chat: ?????
}
}
return state
}
Something like this
const INITIAL_STATE = {
chat: []
}
export default (state = INITIAL_STATE, action) => {
switch (action.type) {
case ADD_NEW_THREAD:
const chat = state.chat.slice();
const thread = chat[action.index].thread.concat(action.thread);
chat.splice(action.index, 1, thread);
return {
chat
};
}
return state
}
I'm having an issue with the sort() in ranking data from coinmarketcap api. With an ajax api call, sort works in returning an array with the right ranking. With an axios api call, seen below, it doesn't.
Here is my code using axios and vue.js:
let coinMarket = 'https://api.coinmarketcap.com/v2/ticker/?limit=10'
let updateInterval = 60 * 1000;
let newApp = new Vue({
el: '#coinApp',
data: {
// data within an array
results: []
},
methods: {
getCoins: function() {
axios
.get(coinMarket)
.then((resp) => {
this.results = formatCoins(resp);
});
},
getColor: (num) => {
return num > 0 ? "color:green;" : "color:red;";
},
},
created: function() {
this.getCoins();
}
})
setInterval(() => {
newApp.getCoins();
},
updateInterval
);
function formatCoins(res) {
var coinDataArray = []
Object.keys(res.data).forEach(function(key) {
coinDataArray.push(res.data[key])
})
coinDataArray.sort(function(a,b) {
return a.rank > b.rank
})
console.log(coinDataArray)
}
Where am I going wrong?
If you look into the data responded by https://api.coinmarketcap.com/v2/ticker/?limit=10, you will find the data you need is under res.data.data, not res.data.
So within the function=formatCoins, replace res.data with res.data.data, then works.
Vue.config.productionTip = false
let coinMarket = 'https://api.coinmarketcap.com/v2/ticker/?limit=10'
let updateInterval = 60 * 1000;
function formatCoins(res) {
var coinDataArray = []
Object.keys(res.data.data).forEach(function(key) {
coinDataArray.push(res.data.data[key])
})
coinDataArray.sort(function(a,b) {
return a.rank - b.rank
})
return coinDataArray
}
let newApp = new Vue({
el: '#coinApp',
data: {
// data within an array
results: []
},
methods: {
getCoins: function() {
axios
.get(coinMarket)
.then((resp) => {
this.results = formatCoins(resp);
});
},
getColor: (num) => {
return num > 0 ? "color:green;" : "color:red;";
},
},
created: function() {
this.getCoins();
}
})
setInterval(() => {
newApp.getCoins();
},
updateInterval
);
<script src="https://cdnjs.cloudflare.com/ajax/libs/axios/0.18.0/axios.js"></script>
<script src="https://unpkg.com/vue#2.5.16/dist/vue.js"></script>
<div id="coinApp">
<div v-for="(record, index) in results" :key="index">
{{index}} - {{record.name}}: {{record.rank}}
</div>
</div>