Google Users: List users data of a specific group - google-api

I am in the need of listing the users data belonging to a specific group within the organization. The documentation does not specify if this is possible. I was really hoping there could be some kind of query that would allow this. For example email in (1#domain.com,2#domain.com). However, I don't see that being possible. The only way I could think to accomplish this would be:
Get a list of all the members in the group (https://developers.google.com/admin-sdk/directory/reference/rest/v1/members/list)
Get each user data by email (https://developers.google.com/admin-sdk/directory/reference/rest/v1/users/get)
The problem with the above approach is that if a group contains 50+ members, this means that I have to make all that amount of requests, which is counter productive. Imagine how long that would take.
Any ideas? Greatly appreciate it.

Unfortunately I don’t think you can skip this two step process, but you can speed it up using batch requests. This
allows you to request up to 1000 calls in a single request. The steps would be:
Make a batch request to get all the members of all the groups you want (using members.list).
Make a batch request to get all the user info that you need using their id (using user.get).
Notice that the data in the result won’t be sorted, but they will be tagged by Content-ID.
References
Sending Batch Requests (Directory API)
Method: members.list (Directory API)
Method: users.get (Directory API)

I thought about the batching request a couple of hours after I posted the question. The problem with Node JS is that it does not has built in support for batch requests, unlike the php client library for example; Therefore, I had to spent some time implementing support for it on my own since I was not able to find any example. I'll share the solution in case it helps someone else or for my future reference.
async function getGroupMembersData(){
const groupEmail = "group#domain.com"; //google group email
const groupMembers = await getGroupMembers(groupEmail).catch(error=>{
console.error(`Error querying group members: ${error.toString()}`);
});
if(!groupMembers){ return; }
const url = "https://www.googleapis.com/batch/admin/directory_v1";
const scopes = ["https://www.googleapis.com/auth/admin.directory.user.readonly"];
const requests = [];
for(let i=0; i<groupMembers.length; ++i){
const user = groupMembers[i];
const request = {
email: user,
endpoint: `GET directory_v1/admin/directory/v1/users/${user}?fields=*`
};
requests.push(request);
}
const batchRequestData = await batchProcess(url, scopes, requests).catch(error=>{
console.error(`Error processing batch request: ${error.toString()}`);
});
if(!batchRequestData){ return; }
const usersList = batchRequestData.map(i=>{
return i.responseBody;
});
console.log(usersList);
}
//get group members using group email address
async function getGroupMembers(groupKey){
const client = await getClient(scopes); //function to get an authorized client, you have to implement on your own
const service = google.admin({version: "directory_v1", auth: client});
const request = await service.members.list({
groupKey,
fields: "members(email)",
maxResults: 200
});
const members = !!request.data.members ? request.data.members.map(i=>i.email) : [];
return members;
}
//batch request processing in groups of 100
async function batchProcess(batchUrl, scopes, requests){
const client = await getClient(scopes); //function to get an authorized client, you have to implement on your own
let results = [];
const boundary = "foobar99998888"; //boundary line definition
let batchBody = ""; const nl = "\n";
const batchLimit = 100; //define batch limit (max supported = 100)
const totalRounds = Math.ceil(requests.length / batchLimit);
let batchRound = 1;
let batchItem = 0;
let roundLimit = batchLimit;
do{
roundLimit = roundLimit < requests.length ? roundLimit : requests.length;
//build the batch request body
for(batchItem; batchItem<roundLimit; batchItem++){
const requestData = requests[batchItem];
batchBody += `--${boundary}${nl}`;
batchBody += `Content-Type: application/http${nl}`;
batchBody += `Content-Id: <myapprequest-${requestData.email}>${nl}${nl}`;
batchBody += `${requestData.endpoint}${nl}`;
}
batchBody += `--${boundary}--`;
//send the batch request
const batchRequest = await client.request({
url: batchUrl,
method: "POST",
headers: {
"Content-Type": `multipart/mixed; boundary=${boundary}`
},
body: batchBody
}).catch(error=>{
console.log("Error processing batch request: " + error);
});
//parse the batch request response
if(!!batchRequest){
const batchResponseData = batchRequest.data;
const responseBoundary = batchRequest.headers["content-type"].split("; ")[1].replace("boundary=", "");
const httpResponses = batchResponseParser(batchResponseData, responseBoundary);
results.push(...httpResponses);
}
batchRound++;
roundLimit += batchLimit;
} while(batchRound <= totalRounds);
return results;
};
//batch response parser
function batchResponseParser(data, boundary){
const nl = "\r\n";
data = data.replace(`--${boundary}--`,"");
const responses = data.split(`--${boundary}`);
responses.shift();
const formattedResponses = responses.map(i=>{
const parts = i.split(`${nl}${nl}`);
const responseMetaParts = (parts[0].replace(nl, "")).split(nl);
let responseMeta = {};
responseMetaParts.forEach(part=>{
const objectParts = part.split(":");
responseMeta[objectParts[0].trim()] = objectParts[1].trim();
});
const responseHeadersParts = parts[1].split(nl);
let responseHeaders = {};
responseHeadersParts.forEach(part=>{
if(part.indexOf("HTTP/1.1") > -1){
responseHeaders.status = part;
} else {
const objectParts = part.split(":");
responseHeaders[objectParts[0].trim()] = objectParts[1].trim();
}
});
const reg = new RegExp(`${nl}`, "g");
const responseBody = JSON.parse(parts[2].replace(reg, ""));
const formatted = {
responseMeta: responseMeta,
responseHeaders: responseHeaders,
responseBody: responseBody
};
return formatted;
});
return formattedResponses;
}

Related

Ethers.js pancakeswap swapExactTokensForTokens invalid response - sendTransaction

I'm trying to execute an Pancakeswap swapExactTokensForTokens using ethers.js but i just keep getting the error invalid response - sendTransaction. Unfortunatly the error doesnt contain any more usefull information then that :(
My code:
const provider = new ethers.providers.WebSocketProvider(config.network);
const tradeWallet = ethers.Wallet.fromMnemonic(config.mnemonic);
const account = tradeWallet.connect(provider);
const router = new ethers.Contract(
'0x10ED43C718714eb63d5aA57B78B54704E256024E',
[
'function getAmountsOut(uint amountIn, address[] memory path) public view returns (uint[] memory amounts)',
'function swapExactTokensForTokens(uint amountIn, uint amountOutMin, address[] calldata path, address to, uint deadline) external returns (uint[] memory amounts)'
],
account
);
[snip]
var amountIn = ethers.utils.parseUnits('0.001', 'ether');
var tokenIn = '0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c';
var tokenOut = '0xd2de3fd31b5c9e1557cf329032615a2870a29ccd';
var gasPrice = '5000000000';
var gasLimit = '231795'
var amounts = await router.getAmountsOut(amountIn, [tokenIn, tokenOut])
const amountOutMin = amounts[1].sub(amounts[1].div(10));
// values at the time where:
// tokenIn: 100000000000000 0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c (WBNB)
// tokenOut: 1810636794711288351 0xd2de3fd31b5c9e1557cf329032615a2870a29ccd
var tx = router.swapExactTokensForTokens(
amountIn,
amountOutMin,
[tokenIn, tokenOut],
addresses.recipient,
Date.now() + 1000 * 60 * 3, //10 minutes
{ gasPrice: gasPrice,
gasLimit: gasLimit
}
);
const receipt = await tx.wait();
use 'swapExactTokensForETHSupportingFeeOnTransferTokens',Because your 'tokenOut' token has a tax function

Optimising Google Speed Insights API Script in Google Sheets

I have created a script in Google Apps Scripts and Google Sheets that returns some speed metrics from the urls that are pasted in the sheet.
The script works good, the only problem is that it takes forever to present the results in the sheet. It makes a call for each url, I suspect that's why it's slow.
Is there any way I can optimise this script so it gives me the results faster?
Screenshot
The code:
const sheet = SpreadsheetApp.getActiveSpreadsheet();
const API_STRING = sheet.getSheetByName("instructions").getRange("K10").getValues();
const PLATFORM = sheet.getSheetByName("urls").getRange("B1").getValues();
const OUTPUT_CELL = sheet.getSheetByName("urls").getRange("B5:" + ("K" + sheet.getLastRow()));
console.log(PLATFORM);
// KPI
const lighthouseMetrics = [
"first-contentful-paint",
"largest-contentful-paint",
'interactive',
"cumulative-layout-shift",
"speed-index",
"total-blocking-time"
]
const fieldData = [
"FIRST_CONTENTFUL_PAINT_MS",
"LARGEST_CONTENTFUL_PAINT_MS",
"FIRST_INPUT_DELAY_MS",
"CUMULATIVE_LAYOUT_SHIFT_SCORE"
]
// CALLING FUNCTION
async function fetch_array() {
let URLS_LIST = sheet.getSheetByName("urls").getRange("A5:" + ("A" + sheet.getLastRow())).getValues();
console.log(URLS_LIST)
let arrayData = [];
for (let element of URLS_LIST) {
let dataEl = await getPageSpeedInfo(PLATFORM, element);
let dataRow = produceArray(dataEl);
arrayData.push(dataRow);
}
return OUTPUT_CELL.setValues(arrayData);
}
// PRODUCE ARRAY WITH KPIS
function produceArray(data) {
let kpiArray = [];
fieldData.forEach(function(item) {
let fieldDataRoute = data.loadingExperience.metrics[item].category;
kpiArray.push(fieldDataRoute);
})
lighthouseMetrics.forEach(function(item) {
let lighthouseRoute = data.lighthouseResult.audits[item].displayValue;
kpiArray.push(lighthouseRoute);
})
return kpiArray;
}
// CALL TO API
async function getPageSpeedInfo(strategy, element) {
let pageSpeedUrl = 'https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=' + element + '&key=' + API_STRING + '&strategy=' + strategy;
console.log(pageSpeedUrl);
let response = await UrlFetchApp.fetch(pageSpeedUrl);
let data = await response.getContentText();
return JSON.parse(data);
}

Is there a way to speed up/batch Google Calendar read/writes?

I am new to Google Apps Script and learning javascript as I go about this project. Over the course of the introductory codelabs I noted the best practice to read all the data into an array with one command, perform operations, and then write it with one command.
I understood how to do this working with Google Sheets but how do I achieve this working with Google Calendar? I have come across a few links discussing batching with Google Calendar API and Advanced Google Services but I didn't understand how to make use of the information.
I basically hope to batch edit events instead of accessing Google Calendar repeatedly in a for loop.
function deleteMonth() {
// Set Date range to delete
var today = new Date();
var firstDay = new Date(today.getFullYear(), today.getMonth(), 1);
var lastDay = new Date(today.getFullYear(), today.getMonth() + 1, 0);
// read spreadsheet data and get User Info from ss
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var idSheet = spreadsheet.getSheetByName('User Info');
//Get users from sheet in array of objects with properties from column header in
//'User Info' (name, email, year, calName, calID, early, late)
var userInfo = getSheetData(idSheet);
var deletedNames = "";
for (i = 0; i < userInfo.length; i++) {
var calID = userInfo[i].calID;
// if we have calID proceed to delete events
if (calID) {
console.time("get events");
var calendar = CalendarApp.getCalendarById(calID);
var events = calendar.getEvents(firstDay, lastDay);
console.timeEnd("get events");
// Delete events and add deleted name to string
// deletedNames
for (i = 0; i < events.length; i++) {
console.time("delete event");
deletedNames += events[i].getTitle() + ", ";
events[i].deleteEvent();
console.timeEnd("delete event");
}
}
}
spreadsheet.toast("Deleted events: \n" + deletedNames);
}
Time output from console.time():
Other related links which sounded relevant:
Using advanced google services (apps script resource)
Google Developer blog?
I believe your goal as follows
You want to delete all events in a month for several calendars using the batch process with Google Apps Script.
You want to reduce the process cost of above process.
For this, how about this answer?
Issue and workaround:
Calendar API can process with the batch requests. The batch requests can run 100 requests by one API call and can process with the asynchronous process. By this, I think that the process cost can bereduced. But, in the current stage, unfortunately, several calendar IDs cannot be used in one batch. When the requests including several calendar IDs, an error of Cannot perform operations on different calendars in the same batch request. occurs. So in your case, all events in a month in one calendar ID can be deleted in one batch requests. It is required to request the number of calendar IDs. I would like to propose this as the current workaround.
By the way, as the modification point of your scrit, in your script, the variable i is used in 1st for loop and 2nd for loop. By this, all values of userInfo are not used. Please be careful this.
Sample script:
Before you run the script, please enable Calendar API at Advanced Google services.
function deleteMonth() {
var today = new Date();
var firstDay = new Date(today.getFullYear(), today.getMonth(), 1);
var lastDay = new Date(today.getFullYear(), today.getMonth() + 1, 0);
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
var idSheet = spreadsheet.getSheetByName('User Info');
var userInfo = getSheetData(idSheet);
var deletedNames = "";
var requests = []; // For batch requests.
for (i = 0; i < userInfo.length; i++) {
var req = [];
var calID = userInfo[i].calID;
if (calID) {
var calendar = CalendarApp.getCalendarById(calID);
var events = calendar.getEvents(firstDay, lastDay);
for (j = 0; j < events.length; j++) {
deletedNames += events[j].getTitle() + ", ";
var e = events[j];
req.push({
method: "DELETE",
endpoint: `https://www.googleapis.com/calendar/v3/calendars/${calID}/events/${e.getId().replace("#google.com", "")}`,
});
}
}
requests.push(req);
}
// Run batch requests.
requests.forEach(req => {
const limit = 100;
const split = Math.ceil(req.length / limit);
const boundary = "xxxxxxxxxx";
for (let i = 0; i < split; i++) {
const object = {batchPath: "batch/calendar/v3", requests: req.splice(0, limit)};
const payload = object.requests.reduce((s, e, i) => s += "Content-Type: application/http\r\nContent-ID: " + i + "\r\n\r\n" + e.method + " " + e.endpoint + "\r\nContent-Type: application/json; charset=utf-8\r\n\r\n" + JSON.stringify(e.requestBody) + "\r\n--" + boundary + "\r\n", "--" + boundary + "\r\n");
const params = {method: "post", contentType: "multipart/mixed; boundary=" + boundary, payload: payload, headers: {Authorization: "Bearer " + ScriptApp.getOAuthToken()}, muteHttpExceptions: true};
var res = UrlFetchApp.fetch("https://www.googleapis.com/" + object.batchPath, params);
console.log(res.getContentText())
}
})
spreadsheet.toast("Deleted events: \n" + deletedNames);
}
Note:
Please use this script with V8.
References:
Advanced Google services
Sending Batch Requests
Events: delete

what is the faster and best way to make a few million SOAP requests and save the results into SqlDb

I have a million records in my table. I want to call a soap service and i need to do process in all the records in less than one hour. and besides i should update my table , insert the requests and responses in my other tables. but the code below works on less than 10 records every time i run my app.
I know My code is wrong,, I want to know what is the best way to do it.
static async Task Send( )
{
var results = new ConcurrentDictionary<string, int>();
using (AppDbContext entities = new AppDbContext())
{
var List = entities.Request.Where(x => x.State == RequestState.InitialState).ToList();
Parallel.ForEach(Enumerable.Range(0, List.Count), async index =>
{
var selected = List.FirstOrDefault();
List.Remove( selected );
var res1 = await DoAsyncJob1(selected); ///await
// var res = CallService(selected);
var res2 = await DoAsyncJob2(selected); ///await
var res3 = await DoAsyncJob3(selected); ///await
// var responses = await Task.WhenAll(DoAsyncJob1, DoAsyncJob2, DoAsyncJob3);
// results.TryAdd(index.ToString(), res);
});
}
}
static async Task<int> DoAsyncJob1(Request item)
{
using (AppDbContext entities = new AppDbContext())
{
var bReq = new BankRequest();
bReq.Amount = Convert.ToDecimal(item.Amount);
bReq.CreatedAt = DateTime.Now;
bReq.DIBAN = item.DIBAN;
bReq.SIBAN = item.SIBAN;
entities.BankRequest.Add(bReq);
entities.SaveChanges();
}
return item.Id;
}
static async Task<int> DoAsyncJob2(Request item)
{
using (AppDbContext entities = new AppDbContext())
{
}
return item.Id;
}
static async Task<int> DoAsyncJob3(Request item)
{
using (AppDbContext entities = new AppDbContext())
{
}
return item.Id;
}
Maybe the below lines are wrong :
var selected = List.FirstOrDefault();
List.Remove( selected );
Thanks in advance..
First, it is a bad practice to use async-await within Parallel.For - you introduce only more load to the task scheduler and more overhead.
Second, you are right:
var selected = List.FirstOrDefault();
List.Remove( selected );
is very, very wrong. Your code will behave in a totally unpredictable way, due to the race conditions.

502 on AWS Cognito ListUsers with PaginationToken

Using AWSMibileHub to set up my backend. Working on an admin page that shows all the users from AWS Cognito.
I’m using ListUsers to get user list and noticed 502 error on the request with a PagenationToken and it seems to be happening randomly.
In CloudWatch shows this error below.
InvalidParameterException: 1 validation error detected: Value 'CAISlAIIARLtAQgDEugBADSdLB5dZXQEaQjoL8y8CE1RGGT3PZ4FpCqxFwJkNuhIeyJAbiI6IlBhZ2luYXRpb25Db250aW51YXRpb25EVE8iLCJuZXh0S2V5IjoiQUFBQUFBQUFCZHM4QVFFQmJBZmU5OFgzUnJTM1BHcnYzVmRiVWNScVdIck82VXFZaTdlZklleVRCSEZsYm1ZN01UVTRPV1k0TVRNdE9UYzVZUzAwWTJKakxUazROekl0TkdRek9UYzROMlpoWmpVM093PT0iLCJwcmV2aW91c1JlcXVlc3RUaW1lIjoxNTQzMDkzODIyNDk5fRogNk7FThSKBOuwQGi DoZBnmNN85UY5oFiSAbHWfOzreY=' at 'paginationToken' failed to satisfy constraint: Member must satisfy regular expression pattern: [\S]+
Does anybody have the same issue or any idea how to fix this issue?
Store the pagination Token in a variable and then add this .replace(/\ /g,'+') to the variable to change the space back to +.
e.g:
const pagination = 'TOKEN FROM COGNITO'.replace(/\ /g,'+')
The pagination_token consists of non alphanumeric characters, so its better to encrypt and decrypt when required.
Encrypt?decrypt utility file
const crypto = require("crypto");
const algorithm = "aes-256-ctr";
const password = "sassed";
class Util {
static encrypt_aes(text) {
let cipher = crypto.createCipher(algorithm, password);
let crypted = cipher.update(text, "utf8", "hex");
crypted += cipher.final("hex");
return crypted;
}
static decrypt_aes(text) {
let decipher = crypto.createDecipher(algorithm, password);
let dec = decipher.update(text, "hex", "utf8");
dec += decipher.final("utf8");
return dec;
}
}
module.exports = Util;
PaginationToken encryption/decription
const Util = require("./utils");
class Users {
static async getUsersAttributes(params) {
let listparams = {
UserPoolId: userPoolID /* required */,
Limit: params.limit || 10,
};
if (params.paginationToken) {
listparams.PaginationToken = Util.decrypt_aes(params.paginationToken);
}
let response = await cognitoISPClient.listUsers(listparams).promise();
response.PaginationToken = Util.encrypt_aes(response.PaginationToken);
return response;
}
}

Resources