Setting page and form with casperjs - casperjs

Hello i need to scrape the result information in my site with browser automation. I have this script:
var casper = require('casper').create();
console.log("casper create OK");
casper.start("https://portale.spefin.it/anagraph/legalperson/atc", function() {
console.log("Connexion URL OK");
// set a wait condition to make sure the page is loaded (particularly iframe in my case)
//fill out the form
this.fillSelectors("form[name='login']",{
'input#username' : "XXXXXXXXX",
'input#pw' : "XXXXXXXX"
});
console.log("Renseignement login et pass OK");
// click the login button
this.click("button[type='submit']");
console.log("Passage bouton login OK");
// switch to iframe (won't be necessary for most)
this.page.switchToChildFrame('https://portale.spefin.it/anagraph/legalperson/atc');
console.log("Switch page OK");
this.wait(5000,function(){
console.log("Attente 5 sec OK");
this.fillSelectors("form[name='advancedFilterForm']",{
'input#tax_code' : "11057560150"
});
console.log("partita iva ok!");
// Test d'une zone sur la page pour valider la connexion
//casper.waitForSelector('.area-status', function() {
//console.log("Validation element sur la page OK");
//});
});
});
The problem is the page is not set and the form is not found....please help me!

first of all I would recommend using verbose and logLevel for testing like so:
var casper = require('casper').create({
verbose: true,
logLevel: "debug"
});
Also you could use the then promise more often. Trust me it helps a bunch.
Using waitForSelector for the rescue.
Bellow find a different version of your code.
casper.start("https://portale.spefin.it/anagraph/legalperson/atc").then(function(){
this.waitForSelector("form[name='login']", function(){
/*form found*/
//the [true] parameter will submit the form, no need for the button click
this.fillSelectors("form[name='login']",{
'input#username' : "XXXXXXXXX",
'input#pw' : "XXXXXXXX"
}, true);
this.then(function(){
//possibly no need for the wait but...
//if you really want to use it
this.wait(5000,function(){
//In my honest opinion you don't need the [switchToChildFrame]
//but instead use [waitForSelector]
this.waitForSelector("form[name='advancedFilterForm']", function(){
/*form found*/
//the [true] parameter will submit the form
this.fillSelectors("form[name='advancedFilterForm']",{
'input#tax_code' : "11057560150"
}, true);
this.then(function(){
this.waitForSelector(".area-status", function(){
/*element found*/
//do what you must perhaps get the info
require('utils').dump(this.getElementInfo('.area-status'));
}, function(){
/*element not found*/
})
});
},function(){
/*could not find advancedFilterForm*/
})
});
});
},function(){
/*form not found*/
});
});
Note: This code wasn't tested.
Hope it helps. :)
Good Scrapings

Related

Pass DataTable reference to the callback function on load

My current code is:
var CommissionLogs = $("#CommissionLogs").DataTable({
ajax: {
url: ajaxurl + '?action=pos&post_action=get_commissions'
},
'initComplete': function (settings, json){
//possible to access 'this'
this.api().columns(1);
}
});
I improved the code above as below with help :
var CommissionLogs = $("#CommissionLogs").DataTable({
ajax: {
url: ajaxurl + '?action=pos&post_action=get_commissions'
},
'initComplete': function(settings, json){
callbackFunction(settings);
}
});
function callbackFunction(settings){
var api = new $.fn.dataTable.Api( settings );
// api is accessible here.
}
Update :
Now I can access api from callback function. But I want use same callback with load() as below code.
CommissionLogs.ajax.url( newAjaxURL ).load( callbackFunction(), true);
But settings param is not accessible in load function.
I can clear and destroy datatable and re initialize always. But what will be the right way.
I think you need settings:
https://datatables.net/reference/type/DataTables.Settings
$('#example').dataTable( {
"initComplete": function(settings, json) {
myFunction(settings);
}
});
function myFunction(settings){
var api = new $.fn.dataTable.Api( settings );
// Output the data for the visible rows to the browser's console
// You might do something more useful with it!
console.log( api.rows( {page:'current'} ).data() );
}
Other option is re-use your var CommissionLogs variable throughout the code without using this, I recommend strongly this last option.
The dataTable.ajax.url().load() has not access to settings.
So can not call a callback function with settings.
But possible to use callback function without settings.
So here is an alternative way to use settings.
CommissionLogs.clear();// clear the table
CommissionLogs.destroy();// destroy the table
CommissionLogs = $("#CommissionLogs").DataTable({
ajax: {
url: newAjaxUrl
},
'initComplete': function (settings, json){
callbackDatatableFunciton(settings);
}
});

CasperJs Google login

I having been working on some code to access my google CSE
For that I need to sign in with my google account.
I have the following code:
var casper = require('casper').create({
verbose: true,
logLevel: 'debug',
waitTimeout: 5000,
clientScripts: ["libs/jquery.min.js"],
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5)
AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4'
});
const google_email = "MY_EMAIL";
const google_passwd = "MY_PASSWORD";
const loginUrl = 'https://accounts.google.com';
casper.start(loginUrl, function() {
this.waitForSelector('#view_container > form');
});
casper.then(function() {
this.fillSelectors('#view_container > form', {
'input[name="identifier"]': google_email
}, false);
});
casper.then(function() {
this.click('#identifierNext');
});
casper.wait(500, function() { //Wait for next page to load
this.capture('images/step01.png');
});
casper.then(function() {
this.evaluate(function () {
var identifierNext = $('#identifierNext');
identifierNext.click();
});
});
casper.then(function() {
this.capture('images/step02.png');
});
casper.run(function() {
this.echo("Done");
});
The part of entering the email seems to work.
But the click part isn't working.
I found this but it seems outdated.
Thanks
I haven't fixed the issue related to the new form, but we found a way to access the old form, so the "old" scripts should still work, and that solution is disabling JS. For that, change the loginUrl to
https://accounts.google.com/ServiceLogin?passive=1209600&continue=https%3A%2F%2Faccounts.google.com%2FManageAccount&followup=https%3A%2F%2Faccounts.google.com%2FManageAccount&flowName=GlifWebSignIn&flowEntry=ServiceLogin&nojavascript=1#identifier
Important thing: nojavascript=1
We are using the script posted in Casperjs Google Login Not working
Just changing the login URL.
Try this - a lot more delays are needed to wait for the dynamically loaded pages.
casper.options.verbose = true; // verbose reporting
casper.options.logLevel = 'debug'; // full log reporting
casper.options.exitOnError = false; // Keep going on error
const google_email = "EMAIL";
const google_passwd = "PASSWORD";
const loginUrl = 'https://accounts.google.com';
// Load the login page
casper.start(loginUrl, function() {
this.waitForSelector('#view_container'); // '> form' doesn't seem to work
});
// Fill in the 'username' form
casper.then(function() {
this.fill('form', {
identifier: google_email,
});
this.sendKeys('#view_container', casper.page.event.key.Enter , {keepFocus: true});
});
// First 'Enter' is too quick for Google, send another one after a pause
casper.wait(2500, function() {
this.sendKeys('#identifierId', casper.page.event.key.Enter , {keepFocus: true});
});
// Wait for the 'password' form
casper.waitForSelector("#passwordNext", function() {
this.echo("password form is apparently available");
});
// Password form seems to load slowly, even if the selector is found/visible, this delay ensures next form fill works
casper.wait(2500, function() {
this.echo("password form is really available");
});
// Fill in the 'password' form
casper.then(function() {
this.fill('form', {
password: google_passwd,
});
this.sendKeys('#view_container', casper.page.event.key.Enter , {keepFocus: true});
});
// First 'Enter' is too quick for Google, send another one after a pause
casper.wait(500, function() {
this.sendKeys('input.whsOnd.zHQkBf', casper.page.event.key.Enter , {keepFocus: true});
});
// Extend timeout to allow for slow dynamic page rendering
casper.options.waitTimeout = 25000;
casper.waitForSelector("#gb", function() {
this.echo("login complete");
});
casper.thenOpen('https://(google app you want)', function() {
// Check it opened okay
});

How to post information in CasperJS

The following is the html I wanna tackle with.
I wanna post a query and see the weather reports for that city. I tried my codes:
var casper = require('casper').create();
var utils = require('utils');
casper.start('http://www.weather.com.cn/weather1d/101210101.shtml');
casper.waitForSelector('input', function()
{
this.fillXPath('div.search clearfix',{'//input[#id="txtZip"]':'shijiazhuang'},true);
});
casper.then(function()
{
utils.dump(this.getTitle());
});
casper.run();
It did not print the web paeg title on the console. I also tried this:
casper.waitForSelector('input', function()
{
this.fillSelectors('div.search clearfix',{'input[id="txtZip"]':'shijiazhuang'},true);
});
}
);
I did not get any web page title also. I got quite confused and did not what I had done wrong. Besides, it seems that this.fill method only tackles with name attributes to post information on CasperJS's official website. I need your help.
CasperJS script:
var casper = require('casper').create(), utils = require('utils');
casper
.start('http://www.weather.com.cn/weather1d/101210101.shtml',function(){
this
.wait(3000,function(){
this.capture('search.png');
utils.dump(this.getTitle());
})
.evaluate(function(){
document.getElementById('txtZip').value='shijiazhuang';
document.querySelector('input[type="button"]').click();
})
})
.run();
Result:
"【石家庄天气】石家庄今天天气预报,今天,今天天气,7天,15天天气预报,天气预报一周,天气预报15天查询"
search.png
You are doing it correct, you only have to modify your script a little bit.
Just modify the selector of the input field the actual don't seems to work, then you should get the correct result.
"this.fillXPath" only fills up the form and don't post it.
...
// fill the form
casper.waitForSelector('input', function() {
this.fillXPath('#txtZip', {
'//input[#id="txtZip"]': 'shijiazhuang'
}, true);
});
// trigger - post the form
casper.then(function() {
this.click('#btnZip');
});
// i can't speak chinese - wait for some specific change on this page
casper.wait(5000);
// take a screenshot of it
casper.then(function() {
this.capture("shijiazhuang_weather.png");
});
...

angularjs make serial actions by condition

i'd like to do user friendly action at my app.
The issue is:
I have one button. When user 'mousedown' on it, i send ajax query to server to get some info. What i want is to show that info only after user 'mouseup' on this button. In other words, when he releases mouse button.
The problem is that user can release mouse button before or after server answers to my ajax call, by i still want to show info only after release button.
I think i can do this with deferred stuff, by i don't have a lot experience in this. Can you give me some info, how to do this?
Example code:
.html
<input type='button' ng-start='start()' ng-end='end()' >
.js
$scope.start = function(){
$http.get(url, data).success(function(result){})
}
$scope.end = function(){
//show result from first function
}
Tried to do it with use of $q, but it appears that $q is not necessary.
.html
<input type='button' ng-mousedown='start()' ng-mouseup='end()' >
<div ng-show="loaded && resolved" ng-bind="processedResult"></div>
.js
$scope.start = function(){
$scope.resolved = false;
$scope.loaded = false;
$http.get(url, data).success(function(result){
$scope.loaded = true;
$scope.processedResult = processResult(result);
});
};
$scope.end = function(){
$scope.resolved = true;
};
I think, i've solved my problem with pretty promise feature.
.html
<input type='button' ng-start='start()' ng-end='end()' >
.js
$scope.start = function() {
promise = $q.when($scope.ApiCall())
.then(function(result){
return result.data;
}, function(error){
console.log(error);
})
;
};
$scope.end = function() {
promise.then(function(result){
processResult(result.datat);
});
};
$scope.ApiCall = function(){
var data = {};
return $http.get(url, {params: data});
}

Running into Error while waiting for Protractor to sync with the page with basic protractor test

describe('my homepage', function() {
var ptor = protractor.getInstance();
beforeEach(function(){
// ptor.ignoreSynchronization = true;
ptor.get('http://localhost/myApp/home.html');
// ptor.sleep(5000);
})
describe('login', function(){
var email = element.all(protractor.By.id('email'))
, pass = ptor.findElement(protractor.By.id('password'))
, loginBtn = ptor.findElement(protractor.By.css('#login button'))
;
it('should input and login', function(){
// email.then(function(obj){
// console.log('email', obj)
// })
email.sendKeys('josephine#hotmail.com');
pass.sendKeys('shakalakabam');
loginBtn.click();
})
})
});
the above code returns
Error: Error while waiting for Protractor to sync with the page: {}
and I have no idea why this is, ptor load the page correctly, it seem to be the selection of the elements that fails.
TO SSHMSH:
Thanks, your almost right, and gave me the right philosophy, so the key is to ptor.sleep(3000) to have each page wait til ptor is in sync with the project.
I got the same error message (Angular 1.2.13). My tests were kicked off too early and Protractor didn't seem to wait for Angular to load.
It appeared that I had misconfigured the protractor config file. When the ng-app directive is not defined on the BODY-element, but on a descendant, you have to adjust the rootElement property in your protractor config file to the selector that defines your angular root element, for example:
// protractor-conf.js
rootElement: '.my-app',
when your HTML is:
<div ng-app="myApp" class="my-app">
I'm using ChromeDriver and the above error usually occurs for the first test. I've managed to get around it like this:
ptor.ignoreSynchronization = true;
ptor.get(targetUrl);
ptor.wait(
function() {
return ptor.driver.getCurrentUrl().then(
function(url) {
return targetUrl == url;
});
}, 2000, 'It\'s taking too long to load ' + targetUrl + '!'
);
Essentially you are waiting for the current URL of the browser to become what you've asked for and allow 2s for this to happen.
You probably want to switch the ignoreSynchronization = false afterwards, possibly wrapping it in a ptor.wait(...). Just wondering, would uncommenting the ptor.sleep(5000); not help?
EDIT:
After some experience with Promise/Deferred I've realised the correct way of doing this would be:
loginBtn.click().then(function () {
ptor.getCurrentUrl(targetUrl).then(function (newURL){
expect(newURL).toBe(whatItShouldBe);
});
});
Please note that if you are changing the URL (that is, moving away from the current AngularJS activated page to another, implying the AngularJS library needs to reload and init) than, at least in my experience, there's no way of avoiding the ptor.sleep(...) call. The above will only work if you are staying on the same Angular page, but changing the part of URL after the hashtag.
In my case, I encountered the error with the following code:
describe("application", function() {
it("should set the title", function() {
browser.getTitle().then(function(title) {
expect(title).toEqual("Welcome");
});
});
});
Fixed it by doing this:
describe("application", function() {
it("should set the title", function() {
browser.get("#/home").then(function() {
return browser.getTitle();
}).then(function(title) {
expect(title).toEqual("Welcome");
});
});
});
In other words, I was forgetting to navigate to the page I wanted to test, so Protractor was having trouble finding Angular. D'oh!
The rootElement param of the exports.config object defined in your protractor configuration file must match the element containing your ng-app directive. This doesn't have to be uniquely identifying the element -- 'div' suffices if the directive is in a div, as in my case.
From referenceConf.js:
// Selector for the element housing the angular app - this defaults to
// body, but is necessary if ng-app is on a descendant of <body>
rootElement: 'div',
I got started with Protractor by watching the otherwise excellent egghead.io lecture, where he uses a condensed exports.config. Since rootElement defaults to body, there is no hint as to what is wrong with your configuration if you don't start with a copy of the provided reference configuration, and even then the
Error while waiting for Protractor to sync with the page: {}
message doesn't give much of a clue.
I had to switch from doing this:
describe('navigation', function(){
browser.get('');
var navbar = element(by.css('#nav'));
it('should have a link to home in the navbar', function(){
//validate
});
it('should have a link to search in the navbar', function(){
//validate
});
});
to doing this:
describe('navigation', function(){
beforeEach(function(){
browser.get('');
});
var navbar = element(by.css('#nav'));
it('should have a link to home in the navbar', function(){
//validate
});
it('should have a link to search in the navbar', function(){
//validate
});
});
the key diff being:
beforeEach(function(){
browser.get('');
});
hope this may help someone.
I was getting this error:
Failed: Error while waiting for Protractor to sync with the page: "window.angular is undefined. This could be either because this is a non-angular page or because your test involves client-side navigation, which can interfere with Protractor's bootstrapping. See http://git.io/v4gXM for details"
The solution was to call page.navigateTo() before page.getTitle().
Before:
import { AppPage } from './app.po';
describe('App', () => {
let page: AppPage;
beforeEach(() => {
page = new AppPage();
});
it('should have the correct title', () => {
expect(page.getTitle()).toEqual('...');
})
});
After:
import { AppPage } from './app.po';
describe('App', () => {
let page: AppPage;
beforeEach(() => {
page = new AppPage();
page.navigateTo();
});
it('should have the correct title', () => {
expect(page.getTitle()).toEqual('...');
})
});
If you are using
browser.restart()
in your spec some times, it throws the same error.
Try to use
await browser.restart()

Resources