Puppeteer, looping through links selected by xpath - xpath

Question

I'm new to puppeteer (and not so great at javascript in general) and am trying to write some basic functionality to:
Get all the links from an XPath
Loop through and click those links
Screenshot and save the HTML of the page
Go back, screenshot and save the HTML of the records page to be save in the same directory of the others and start the process over
The error I get is:
Evaluation failed: DOMException: Failed to execute 'querySelector' on 'Document': '0' is not a valid selector
Here's the code I have:
I'm fairly confident all the code works, except for the issues i'm having getting the right things to click on with my XPath. The website I'm getting these from is:
https://hrlb.oregon.gov/bspa/licenseelookup/searchdir.asp?searchby=lastname&searchfor=a&stateselect=none&Submit=Search
code:
const records = await page.$x('//table[2]//tr[td[a]]//td[1]/a');
let int = 0;
for (let record in records) {
await Promise.all([
page.waitForNavigation(),
page.click(record)
]);
await Promise.all([makeDirectory('screenshots/item'+int), makeDirectory('screenshots/item'+int+'/base'), makeDirectory('screenshots/item'+int+'/record')]);
let recordPath = "screenshots/item"+int+"/record/record.html";
let basePath = "screenshots/item"+int+"/base/base.html";
page.screenshot({path: "screenshots/item"+int+"/record/record.png", fullPage: true});
let recordBody = await page.evaluate(() => document.body.innerHTML);
await saveHtml(recordPath, recordBody);
await Promise.all([
page.waitForNavigation(),
page.goBack()
]);
await page.screenshot({path: "screenshots/item"+int+"/base/base.png", fullPage: true});
let baseBody = await page.evaluate(() => document.body.innerHTML);
await saveHtml(basePath, baseBody);
int++;
console.log(record);
}
async function makeDirectory(path) {
mkdirp(path, function(err) {
if (err) throw err;
});
};
async function saveHtml(path, html) {
await fs.writeFile(path, html, (err) => {
if (err) throw err;
});
};
Note: I'm required to use XPath :(
Updated 6/25/18
This now gives me all the links from the xpath selector. Which i then iterate over and just use a page.goto, to go to the correct site.
const linksXPath = '//table[2]//tr[td[a]]//td[1]/a';
const links = await page.evaluate((selector) => {
let results = [];
let query = document.evaluate(selector,
document,
null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
for (let i=0, length=query.snapshotLength; i<length; ++i) {
results.push(query.snapshotItem(i).href);
}
return results;
}, linksXPath);

Answer 1

I think it is your selector that is the issue.
I believe your selector for the table should be:
"body > table > tbody > tr:nth-child(2) > td > table > tbody > tr:nth-child(1) > td > table.bodytext > tbody"
The easiest way to get the right selector for a page is to use the Chrome Dev Tools.
Inspect the page, and go to the "Elements" tab. From there, you should see all of the HTML elements. Right click on the one you want (I went for <tbody> so you can iterate through the <tr> elements.) and choose copy > copy selector.

Answer 2

My code is now doing what I need it to do, however I wish there was a simpler way of doing this. Also, you'll see as I iterate over the links, i'm using a page.goto function to go there. I still don't know of a way to use page.click. I would have to use the xpath to get all the td's and then click on them, but i was never able to get that to work. So here is the working product:
const puppeteer = require('puppeteer');
const fs = require('fs');
const mkdirp = require('mkdirp');
async function run() {
const pageToClick = 'body > table > tbody > tr:nth-child(3) > td > table > tbody > tr > td > form > table > tbody > tr:nth-child(3) > td > div > input[type="submit"]';
const select = 'body > table > tbody > tr:nth-child(3) > td > table > tbody > tr > td > form > table > tbody > tr:nth-child(1) > td:nth-child(2) > select';
const inputField = 'body > table > tbody > tr:nth-child(3) > td > table > tbody > tr > td > form > table > tbody > tr:nth-child(2) > td:nth-child(2) > input[type="text"]:nth-child(1)';
const linksXPath = '//table[2]//tr[td[a]]//td[1]/a';
const browser = await puppeteer.launch({
headless: true
});
const page = await browser.newPage();
await page.goto('https://hrlb.oregon.gov/bspa/licenseelookup/');
await page.select(select, 'lastname');
await page.focus(inputField);
await page.keyboard.type('a');
await Promise.all([
page.waitForNavigation(),
page.click(pageToClick)
]);
const links = await page.evaluate((selector) => {
let results = [];
let query = document.evaluate(selector,
document,
null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
for (let i=0, length=query.snapshotLength; i<length; ++i) {
results.push(query.snapshotItem(i).href);
}
return results;
}, linksXPath);
const basePic = await page.screenshot({fullPage: true});
let baseBody = await page.evaluate(() => document.body.innerHTML);
let int = 0;
for (i = 0; i < links.length; i++) {
await Promise.all([
page.waitForNavigation(),
page.goto(links[i])
]);
await Promise.all([makeDirectory('screenshots/item'+int), makeDirectory('screenshots/item'+int+'/base'), makeDirectory('screenshots/item'+int+'/record')]);
let recordPath = "screenshots/item"+int+"/record/record.html";
let basePath = "screenshots/item"+int+"/base/base.html";
let basePicPath = "screenshots/item"+int+"/base/base.png";
await page.screenshot({path: "screenshots/item"+int+"/record/record.png", fullPage: true});
let recordBody = await page.evaluate(() => document.body.innerHTML);
await saveFile(recordPath, recordBody);
await Promise.all([
page.waitForNavigation(),
page.goBack()
]);
await saveFile(basePath, baseBody);
await saveFile(basePicPath, basePic);
int++;
}
await page.close();
await browser.close();
}
async function makeDirectory(path) {
mkdirp(path, function(err) {
if (err) throw err;
});
};
async function saveFile(path, html) {
await fs.writeFile(path, html, (err) => {
if (err) throw err;
});
};
run();