Unable to download file with CasperJS - casperjs

Question

I am trying to download a file with CasperJS. If using browser, the download begins when user clicks a button and the response headers look like this:
I have tried these two methods with no luck:
1) https://stackoverflow.com/a/26334034
With this approach, the code block inside the if-statement is never executed. If I remove the condition, a bunch of resources are being saved, such as css files and so on. So the event listener is working, but for some reason not triggering when I use CasperJs click function to click the button that should start the download.
2) https://stackoverflow.com/a/30122021/692695
File.csv is saved but it's the web sites sourcecode, not the csv-file I get when I click the button on the website.
All of my code:
'use strict';
var utils = require('utils');
var casper = require('casper').create({
//verbose: true,
//logLevel: "debug",
clientScripts: ["node_modules/jquery/dist/jquery.min.js"]
});
function writeHtml(filename) {
var fs = require('fs');
var content = casper.getHTML();
fs.write(filename, content, 'w');
}
function getUrl() {
var url;
url = $('.tableofcontent_link:contains("Väestö työmarkkina-aseman, sukupuolen ja iän mukaan")').parent().attr('href');
return url;
}
casper.selectOptionByValue = function(selector, valueToMatch){
this.evaluate(function(selector, valueToMatch){
var select = document.querySelector(selector),
found = false;
Array.prototype.forEach.call(select.children, function(opt, i){
if (!found && opt.value.indexOf(valueToMatch) !== -1) {
select.selectedIndex = i;
found = true;
}
});
// dispatch change event in case there is some kind of validation
var evt = document.createEvent("UIEvents"); // or "HTMLEvents"
evt.initUIEvent("change", true, true);
select.dispatchEvent(evt);
}, selector, valueToMatch);
};
var link;
var url = 'http://pxnet2.stat.fi/PXWeb/pxweb/fi/StatFin/StatFin__tym__tyti/?table';
casper.start(url);
casper.userAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X)');
casper.then(function () {
this.echo("Now at: " + this.getCurrentUrl());
link = 'http://pxnet2.stat.fi' + casper.evaluate(getUrl);
});
casper.then(function () {
this.open(link);
});
casper.then(function() {
this.echo("Now at: " + this.getCurrentUrl());
// Select all data for each item
casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_VariableSelectorValueSelectRepeater_ctl01_VariableValueSelect_VariableValueSelect_SelectAllButton');
casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_VariableSelectorValueSelectRepeater_ctl02_VariableValueSelect_VariableValueSelect_SelectAllButton');
casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_VariableSelectorValueSelectRepeater_ctl03_VariableValueSelect_VariableValueSelect_SelectAllButton');
casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_VariableSelectorValueSelectRepeater_ctl04_VariableValueSelect_VariableValueSelect_SelectAllButton');
casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_VariableSelectorValueSelectRepeater_ctl05_VariableValueSelect_VariableValueSelect_SelectAllButton');
});
casper.then(function() {
// casper.selectOptionByValue('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_OutputFormats_OutputFormats_OutputFormatDropDownList',
// 'FileTypeExcelX');
// Select the format of the file from the select option list at the bottom
casper.selectOptionByValue('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_OutputFormats_OutputFormats_OutputFormatDropDownList',
'FileTypeCsvWithHeadingAndSemiColon');
});
casper.then(function () {
// just for debugging
writeHtml('page1.html');
});
casper.then(function() {
//casper.click('#ctl00_ContentPlaceHolderMain_VariableSelector1_VariableSelector1_ButtonViewTable');
});
casper.then(function() {
// FIRST ATTEMPT TO LOAD THE DATA TO a file called file.csv
var formData = casper.evaluate(function(){
return $('form#aspnetForm').serialize();
});
//this.echo("Params: " + formData);
var targetFile = 'file.csv';
casper.download(link, targetFile, 'POST', formData);
});
casper.then(function () {
// just for debugging
writeHtml('page2.html');
});
// SECCOND ATTEMPT TO LOAD THE DATA TO a file called stats.csv
casper.on('resource.received', function (resource) {
if ((resource.url.indexOf('tyti_001') !== -1) ) {
this.echo(resource.url);
var file;
file = "stats.csv";
try {
this.echo("Attempting to download file " + file);
var fs = require('fs');
casper.download(resource.url, fs.workingDirectory + '/' + file);
} catch (e) {
this.echo(e);
}
}
});
casper.run(function () {
this.echo('End').exit();
});
And my package.json:
{
"scripts": {
"test": "dotest"
},
"pre-commit": ["test"],
"dependencies": {
"jquery": "^3.3.1"
},
"devDependencies": {
"pre-commit": "^1.2.2"
}
}
Explanation of the code:
First visit this page: http://pxnet2.stat.fi/PXWeb/pxweb/fi/StatFin/StatFin__tym__tyti/statfin_tyti_pxt_001.px/?rxid=bd4d5dc1-358d-407e-ae47-13266b79bfd0
There, dynamically pick a specified link and move there.
Select all data by clicking the V-shapen icon (look at the attached screenshot) and then select the format of the file.

Answer 1

I've faced this issue earlier with all versions except phantomjs 2.0.0. I also tried the solutions you shared from SO a year ago and they didn't work as well.
I'm going to assume you're using a phantomjs version other than 2.0.0.
Here's the link to download it
https://bitbucket.org/ariya/phantomjs/downloads/
With it, you will have access to onFileDownload method which you can override and use like below
casper.page.onFileDownload = function(status){
console.log('onFileDownload(' + status + ')');
return "newfile.csv";
};
onFileDownload will be called whenever a file is downloaded as a result of clicking a button (ajax) or as a result of sequential get/post requests.
All you have to do is, trigger the click on the button/link that will initiate download.
Note : My solution is assuming that everything else (site is not blocking phantomjs and your request headers/cookies are as expected)