Is there any cost advantage of Parse.Object.saveAll vs. saving individually? - parse-platform

The Parse JS SDK provides a Parse.Object.saveAll() method to save many objects with one command.
From looking at ParseServerRESTController.js it seems that each object is saved individually:
if (path === '/batch') {
let initialPromise = Promise.resolve();
if (data.transaction === true) {
initialPromise = config.database.createTransactionalSession();
}
return initialPromise.then(() => {
const promises = data.requests.map(request => {
return handleRequest(
request.method,
request.path,
request.body,
options,
config
).then(
response => {
return {
success: response
};
},
error => {
return {
error: {
code: error.code,
error: error.message
},
};
}
);
});
return Promise.all(promises).then(result => {
if (data.transaction === true) {
if (
result.find(resultItem => typeof resultItem.error === 'object')
) {
return config.database.abortTransactionalSession().then(() => {
return Promise.reject(result);
});
} else {
return config.database.commitTransactionalSession().then(() => {
return result;
});
}
} else {
return result;
}
});
});
}
It seems that saveAll is merely a convenience wrapper around saving each object individually, so it still does seem to make n database requests for n objects.
It it correct that saveAll has no cost advantage (performance, network traffic, etc) vs. saving each object individually in Cloud Code?

I can tell you that the answer is that Parse.Object.saveAll and Parse.Object.destroyAll batch requests by default in batches of 20 objects. But why take my word for it? Let's test it out!
Turn verbose logging on and then run the following:
const run = async function run() {
const objects = [...Array(10).keys()].map(i => new Parse.Object('Test').set({i}));
await Parse.Object.saveAll(objects);
const promises = objects.map(o => o.increment('i').save());
return Promise.all(promises);
};
run()
.then(console.log)
.catch(console.error);
And here's the output from the parse-server logs (I've truncated it, but it should be enough to be apparent what is going on):
verbose: REQUEST for [POST] /parse/batch: { // <--- note the path
"requests": [ // <--- an array of requests!!!
{
"method": "POST",
"body": {
"i": 0
},
"path": "/parse/classes/Test"
},
... skip the next 7, you get the idea
{
"method": "POST",
"body": {
"i": 9
},
"path": "/parse/classes/Test"
}
]
}
.... // <-- remove some irrelevent output for brevity.
verbose: RESPONSE from [POST] /parse/batch: {
"response": [
{
"success": {
"objectId": "szVkuqURVq",
"createdAt": "2020-03-05T21:25:44.487Z"
}
},
...
{
"success": {
"objectId": "D18WB4Nsra",
"createdAt": "2020-03-05T21:25:44.491Z"
}
}
]
}
...
// now we iterate through and there's a request per object.
verbose: REQUEST for [PUT] /parse/classes/Test/szVkuqURVq: {
"i": {
"__op": "Increment",
"amount": 1
}
}
...
verbose: REQUEST for [PUT] /parse/classes/Test/HtIqDIsrX3: {
"i": {
"__op": "Increment",
"amount": 1
}
}
// and the responses...
verbose: RESPONSE from [PUT] /parse/classes/Test/szVkuqURVq: {
"response": {
"i": 1,
"updatedAt": "2020-03-05T21:25:44.714Z"
}
}
...
In the core manager code, you do correctly identify that we are making a request for each object to the data store (i.e. MongoDB), This is necessary because an object may have relations or pointers that have to be handled and that may require additional calls to the data store.
BUT! calls between the parse server and the data store are usually over very fast networks using a binary format, whereas calls between the client and the parse server are JSON and go over longer distances with ordinarily much slower connections.
There is one other potential advantage that you can see in the core manager code which is that the batch is done in a transaction.

Related

Slack Bolt App: Options body view state is not updated like actions body view state

I am trying to implement dependent external selects inside a modal but I am having problems passing the state of the first dropdown to the second. I can see the state I need inside the app.action listener but I am not getting the same state inside the app.options listener.
body.view.state inside app.action("case_types"). I specifically need the case_create_case_type_block state.
"state": {
"values": {
"case_create_user_select_block": {
"case_create_selected_user": {
"type": "users_select",
"selected_user": "U01R3AE65GE"
}
},
"case_create_case_type_block": {
"case_types": {
"type": "external_select",
"selected_option": {
"text": { "type": "plain_text", "text": "Incident", "emoji": true },
"value": "Incident"
}
}
},
"case_create_case_subtype_block": {
"case_subtypes": { "type": "external_select", "selected_option": null }
},
"case_create_case_owner_block": {
"case_owners": { "type": "external_select", "selected_option": null }
},
"case_create_subject_block": {
"case_create_case_subject": {
"type": "plain_text_input",
"value": null
}
},
"case_create_description_block": {
"case_create_case_description": {
"type": "plain_text_input",
"value": null
}
}
}
},
body.view.state inside app.options("case_subtypes")
"state": {
"values": {
"case_create_user_select_block": {
"case_create_selected_user": {
"type": "users_select",
"selected_user": "U01R3AE65GE"
}
}
}
},
I did also try to update the view myself hoping it would update the state variables inside app.action({ action_id: "case_types" })
//need to update view with new values
try {
// Call views.update with the built-in client
const result = await client.views.update({
// Pass the view_id
view_id: body.view.id,
// Pass the current hash to avoid race conditions
hash: body.view.hash,
});
console.log("Case Type View Update result:");
console.log(JSON.stringify(result));
//await ack();
} catch (error) {
console.error(error);
//await ack();
}
I ended up posting this on the github issues page for slack bolt. This was a bug that will fixed in a future release. Below is the workaround using private metadata to hold the state to check for future dependent dropdowns.
// Action handler for case type
app.action('case_type_action_id', async ({ ack, body, client }) => {
try {
// Create a copy of the modal view template and update the private metadata
// with the selected case type from the first external select
const viewTemplate = JSON.parse(JSON.stringify(modalViewTemplate))
viewTemplate.private_metadata = JSON.stringify({
case_type: body.view.state.values['case_type_block_id']['case_type_action_id'].selected_option.value,
});
// Call views.update with the built-in client
const result = await client.views.update({
// Pass the view_id
view_id: body.view.id,
// Pass the current hash to avoid race conditions
hash: body.view.hash,
// Pass the updated view
view: viewTemplate,
});
console.log(JSON.stringify(result, 0, 2));
} catch (error) {
console.error(error);
}
await ack();
});
// Options handler for case subtype
app.options('case_subtype_action_id', async ({ body, options, ack }) => {
try {
// Get the private metadata that stores the selected case type
const privateMetadata = JSON.parse(body.view.private_metadata);
// Continue to render the case subtype options based on the case type
// ...
} catch (error) {
console.error(error);
}
});
See the full explaination here: https://github.com/slackapi/bolt-js/issues/1146

Parse Server ignores ACL

I have a simple Parse Cloud Code function for my NewsFeed Objects. For these NewsFeed Objects I set an ACL such that the fromUser can write to it and the toUser can read and write to it. I try to get these objects for every user with the following function:
Parse.Cloud.define("get_feed", async (request) => {
let user = request.user;
var query = new Parse.Query("NewsFeed");
query.equalTo("toUser", user);
query.include("fromUser");
query.descending("createdAt");
let result;
try {
result = await query.find();
} catch (error) {
throw error.message;
}
return result;
});
I would expect that I get all the objects which satisfy the query and have the following ACL:
"ACL" : {
"xXl3OIndCP": {
"read": true,
"write": true
},
"VPuRMZGhcv": {
"write": true
}
}
But unfortunately it works only if I add public read access to the ACL such that it looks like this
"ACL" : {
"*": {
"read": true
},
"xXl3OIndCP": {
"read": true,
"write": true
},
"VPuRMZGhcv": {
"write": true
}
}
I don't think that this is the expected behavior or am I wrong? Did anybody face this issue before?
Well, turned out that one has to pass the session token in the find() function so I had to use result = await query.find({sessionToken : user.getSessionToken()}); instead of result = await query.find();

Apollo Server / GraphQL - Properties of Nested Array Returning Null

Bear with me, I will explain this the best I can. Please let me know if more information is needed, I am trying to keep this as brief as possible.
I am using Apollo Server and the 'apollo-datasource-rest' plugin to access a REST API. When attempting to get the property values from a nested array of objects I get a null response for each field/property. In addition, the array being queried is only showing a single iteration when multiple are available.
The field in question is the 'cores' field within the Rocket type, i.e., launch.rocket.firstStage.cores
I have attempted various ways of mapping through 'cores' (thinking this was what it wanted) with no success.
To keep things short and simple I'm only including the code for the specific issue. All other parts of the query are operating as expected.
You can view the API response I am hitting here: https://api.spacexdata.com/v3/launches/77
schema.js
const { gql } = require('apollo-server');
const typeDefs = gql`
type Query {
singleLaunch(flightNumber: Int!): Launch
}
type Launch {
flightNumber: Int!
rocket: Rocket
}
type Rocket {
firstStage: Cores
}
type Cores {
cores: [CoreFields]
}
type CoreFields {
flight: Int
gridfins: Boolean
legs: Boolean
reused: Boolean
landingType: String
landingVehicle: String
landingSuccess: Boolean
}
`;
module.exports = typeDefs;
Data Source - launch.js
const { RESTDataSource } = require('apollo-datasource-rest');
class LaunchAPI extends RESTDataSource {
constructor() {
super();
this.baseURL = 'https://api.spacexdata.com/v3/';
}
async getLaunchById({ launchId }) {
const res = await this.get('launches', {
flight_number: launchId,
});
return this.launchReducer(res[0]);
}
launchReducer(launch) {
return {
flightNumber: launch.flight_number || 0,
rocket: {
firstStage: {
cores: [
{
flight: launch.rocket.first_stage.cores.flight,
gridfins: launch.rocket.first_stage.cores.gridfins,
legs: launch.rocket.first_stage.cores.legs,
landingType: launch.rocket.first_stage.cores.landing_type,
landingVehicle: launch.rocket.first_stage.cores.landing_vehicle,
landingSuccess: launch.rocket.first_stage.cores.landing_success,
},
],
},
};
}
}
module.exports = LaunchAPI;
resolvers.js
module.exports = {
Query: {
singleLaunch: (_, { flightNumber }, { dataSources }) =>
dataSources.launchAPI.getLaunchById({ launchId: flightNumber }),
},
};
Query
query GetLaunchById($flightNumber: Int!) {
singleLaunch(flightNumber: $flightNumber) {
flightNumber
rocket {
firstStage {
cores {
flight
gridfins
legs
reused
landingType
landingVehicle
landingSuccess
}
}
}
}
}
Expected Result
{
"data": {
"singleLaunch": {
"flightNumber": 77,
"rocket": {
"firstStage": {
"cores": [
{
"flight": 1,
"gridfins": true,
"legs": true,
"reused": true,
"landingType": "ASDS",
"landingVehicle": "OCISLY",
"landSuccess": true,
},
{
"flight": 1,
"gridfins": true,
"legs": true,
"reused": false,
"landingType": "RTLS",
"landingVehicle": "LZ-1",
"landSuccess": true
},
{
"flight": 1,
"gridfins": true,
"legs": true,
"reused": false,
"landingType": "RTLS",
"landingVehicle": "LZ-2",
"landSuccess": true
},
]
}
},
}
}
}
Actual Result (Through GraphQL Playground)
{
"data": {
"singleLaunch": {
"flightNumber": 77,
"rocket": {
"firstStage": {
"cores": [
{
"flight": null,
"gridfins": null,
"legs": null,
"reused": null,
"landingType": null,
"landingVehicle": null,
"landingSuccess": null
}
]
}
},
}
}
}
Any suggestions as to what I am doing wrong here would be greatly appreciated. Again, let me know if more information is needed.
Thank you!
Missing base url
There should be
await this.get( this.baseURL + 'launches'
IMHO there should be a map used within launchReducer to return an array, sth like:
launchReducer(launch) {
return {
flightNumber: launch.flight_number || 0,
rocket: {
firstStage: {
cores: launch.rocket.first_stage.cores.map(core => ({
flight: core.flight,
gridfins: core.gridfins,
legs: core.legs,
landingType: core.landing_type,
landingVehicle: core.landing_vehicle,
landSuccess: core.land_success,
})),
},
},
};
}
.map(core => ({ is for returning object [literal], the same as/shorter version of .map(core => { return {

Graphql returning Cannot return null for non-nullable field Query.getDate. As I am new to graphql I want to know is my approach is wrong or my code?

I have created resolver, schema and handler which will fetch some record from dynamoDB. Now when I perform query then I am getting "Cannot return null for non-nullable field Query.getDate" error. I would like to know whether my approach is wrong or there is any change required in code.
My code : https://gist.github.com/vivek-chavan/95e7450ff73c8382a48fb5e6a5b96025
Input to lambda :
{
"query": "query getDate {\r\n getDate(id: \"0f92fa40-8036-11e8-b106-952d7c9eb822#eu-west-1:ba1c96e7-92ff-4d63-879a-93d5e397b18a\") {\r\n id\r\n transaction_date\r\n }\r\n }"
}
Response :
{
"errors": [
{
"message": "Cannot return null for non-nullable field Query.getDate.",
"locations": [
{
"line": 2,
"column": 7
}
],
"path": [
"getDate"
]
}
],
"data": null
}
Logs of lambda function :
[ { Error: Cannot return null for non-nullable field Query.getDate.
at completeValue (/var/task/node_modules/graphql/execution/execute.js:568:13)
at completeValueCatchingError (/var/task/node_modules/graphql/execution/execute.js:503:19)
at resolveField (/var/task/node_modules/graphql/execution/execute.js:447:10)
at executeFields (/var/task/node_modules/graphql/execution/execute.js:293:18)
at executeOperation (/var/task/node_modules/graphql/execution/execute.js:237:122)
at executeImpl (/var/task/node_modules/graphql/execution/execute.js:85:14)
at execute (/var/task/node_modules/graphql/execution/execute.js:62:229)
at graphqlImpl (/var/task/node_modules/graphql/graphql.js:86:31)
at /var/task/node_modules/graphql/graphql.js:32:223
at graphql (/var/task/node_modules/graphql/graphql.js:30:10)
message: 'Cannot return null for non-nullable field Query.getDate.',
locations: [Object],
path: [Object] } ],
data: null }
2019-02-25T10:07:16.340Z 9f75d1ea-2659-490b-ba59-5289a5d18d73 { Item:
{ model: 'g5',
transaction_date: '2018-07-05T09:30:31.391Z',
id: '0f92fa40-8036-11e8-b106-952d7c9eb822#eu-west-1:ba1c96e7-92ff-4d63-879a-93d5e397b18a',
make: 'moto' } }
Thanks in advance!
This is your code:
const data = {
getDate(args) {
var params = {
TableName: 'delete_this',
Key: {
"id": args.id
}
};
client.get(params, function(err,data){
if(err){
console.log('error occured '+err)
}else{
console.log(data)
}
});
},
};
const resolvers = {
Query: {
getDate: (root, args) => data.getDate(args),
},
};
You're seeing that error because getDate is a a Non-Null field in your schema, but it is resolving to null. Your resolver needs to return either a value of the appropriate type, or a Promise that will resolve to that value. If you change data like this
const data = {
getDate(args) {
return {
id: 'someString',
transaction_date: 'someString',
}
}
}
you'll see the error go away. Of course, your goal is to return data from your database, so we need to add that code back in. However, your existing code utilizes a callback. Anything you do inside the callback is irrelevant because it's ran after your resolver function returns. So we need to use a Promise instead.
While you can wrap a callback with Promise, that shouldn't be necessary with aws-sdk since newer versions support Promises. Something like this should be sufficient:
const data = {
getDate(args) {
const params = //...
// must return the resulting Promise here
return client.get(params).promise().then(result => {
return {
// id and transaction_date based on result
}
})
}
}
Or using async/await syntax:
const data = {
async getDate(args) {
const params = //...
const result = await client.get(params).promise()
return {
// id and transaction_date based on result
}
}
}

How to wait until all bulk writes are completed in elastic search api

Using NodeJS elastic search client. Trying to write a data importer to bulk import documents from MongoDB. The problem I'm having is the index refresh doesn't seem to wait until all documents are written to elastic before checking the counts.
Using the streams API in node to read the records into a batch, then using the elastic API bulk command to write the records. Shown below:
function rebuildIndex(modelName, queryStream, openStream, done) {
logger.debug('Rebuilding %s index', modelName);
async.series([
function (next) {
deleteType(modelName, function (err, result) {
next(err, result);
});
},
function (next) {
var Model;
var i = 0;
var batchSize = settings.indexBatchSize;
var batch = [];
var stream;
if (queryStream && !openStream) {
stream = queryStream.stream();
} else if (queryStream && openStream) {
stream = queryStream;
}else
{
Model = mongoose.model(modelName);
stream = Model.find({}).stream();
}
stream.on("data", function (doc) {
logger.debug('indexing %s', doc.userType);
batch.push({
index: {
"_index": settings.index,
"_type": modelName.toLowerCase(),
"_id": doc._id.toString()
}
});
var obj;
if (doc.toObject){
obj = doc.toObject();
}else{
obj = doc;
}
obj = _.clone(obj);
delete obj._id;
batch.push(obj);
i++;
if (i % batchSize == 0) {
console.log(chalk.green('Loaded %s records'), i);
client().bulk({
body: batch
}, function (err, resp) {
if (err) {
next(err);
} else if (resp.errors) {
next(resp);
}
});
batch = [];
}
});
// When the stream ends write the remaining records
stream.on("end", function () {
if (batch.length > 0) {
console.log(chalk.green('Loaded %s records'), batch.length / 2);
client().bulk({
body: batch
}, function (err, resp) {
if (err) {
logger.error(err, 'Failed to rebuild index');
next(err);
} else if (resp.errors) {
logger.error(resp.errors, 'Failed to rebuild index');
next(resp);
} else {
logger.debug('Completed rebuild of %s index', modelName);
next();
}
});
} else {
next();
}
batch = [];
})
}
],
function (err) {
if (err)
logger.error(err);
done(err);
}
);
}
I use this helper to check the document counts in the index. Without the timeout, the counts in the index are wrong, but with the timeout they're okay.
/**
* A helper function to count the number of documents in the search index for a particular type.
* #param type The type, e.g. User, Customer etc.
* #param done A callback to report the count.
*/
function checkCount(type, done) {
async.series([
function(next){
setTimeout(next, 1500);
},
function (next) {
refreshIndex(next);
},
function (next) {
client().count({
"index": settings.index,
"type": type.toLowerCase(),
"ignore": [404]
}, function (error, count) {
if (error) {
next(error);
} else {
next(error, count.count);
}
});
}
], function (err, count) {
if (err)
logger.error({"err": err}, "Could not check index counts.");
done(err, count[2]);
});
}
And this helper is supposed to refresh the index after the update completes:
// required to get results to show up immediately in tests. Otherwise there's a 1 second delay
// between adding an entry and it showing up in a search.
function refreshIndex(done) {
client().indices.refresh({
"index": settings.index,
"ignore": [404]
}, function (error, response) {
if (error) {
done(error);
} else {
logger.debug("deleted index");
done();
}
});
}
The loader works okay, except this test fails because of timing between the bulk load and the count check:
it('should be able to rebuild and reindex customer data', function (done) {
this.timeout(0); // otherwise the stream reports a timeout error
logger.debug("Testing the customer reindexing process");
// pass null to use the generic find all query
searchUtils.rebuildIndex("Customer", queryStream, false, function () {
searchUtils.checkCount("Customer", function (err, count) {
th.checkSystemErrors(err, count);
count.should.equal(volume.totalCustomers);
done();
})
});
});
I observe random results in the counts from the tests. With the artificial delay (setTimeout in the checkCount function) then the counts match. So I conclude that the documents are eventually written to elastic and the test would pass. I thought the indices.refresh would essentially force a wait until the documents are all written to the index, but it doesn't seem to be working with this approach.
The setTimeout hack is not really sustainable when the volume goes to actual production level....so how can I ensure the bulk calls are completely written to elastic index before checking the count of documents?
Take a look at the "refresh" parameter (elasticsearch documentation)
For example:
let bulkUpdatesBody = [ bulk actions / docs to index go here ]
client.bulk({
refresh: "wait_for",
body: bulkUpdatesBody
});
I'm not sure if this is the answer or not - but I flushed the index prior to checking the count. It "appears" to work, but I don't know if it's just because of the timing between the calls. Perhaps someone from elastic team knows if flushing the index will really solve the issue?
function checkCount(type, done) {
async.series([
function(next) {
client().indices.flush({
"index": settings.index,
"ignore": [404]
}, function (error, count) {
if (error) {
next(error);
} else {
next(error, count.count);
}
});
},
function (next) {
refreshIndex(type, next);
},
function (next) {
client().count({
"index": settings.index,
"type": type.toLowerCase(),
"ignore": [404]
}, function (error, count) {
if (error) {
next(error);
} else {
next(error, count.count);
}
});
}
], function (err, count) {
if (err)
logger.error({"err": err}, "Could not check index counts.");
done(err, count[2]);
});
}

Resources