Curl with NodeJS

Curl with NodeJS

var fs = require(‘fs’);
var request = require(‘request’);
for (var i = 0; i < urls.length; i++) {
    var file = ‘log’+[i]+‘.txt’;
    var url = urls[i];
    console.log(url);
    console.log(file);
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(‘request url: ‘+url);
console.log(‘request file: ‘+file);
fs.writeFile(file, body);
}
});
}
/*
Output:
log0.txt
log1.txt
request url: http://www.bing.com
request file: log1.txt
request url: http://www.bing.com
request file: log1.txt
*/

 

var fs = require(‘fs’);
var request = require(‘request’);
function scrape(url,file){
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(‘request url: ‘+url);
console.log(‘request file: ‘+file);
fs.writeFile(file, body);
}
});
}
for (var i = 0; i < urls.length; i++) {
    var file = ‘log’+[i]+‘.txt’;
    var url = urls[i];
    console.log(url);
    console.log(file);
    scrape(url,file);
}
/*
log0.txt
log1.txt
request url: http://www.bing.com
request file: log1.txt
request file: log0.txt
*/

Another good nodeJS module for doing curl like web scraping iscurlrequest.  PhantomJS.

 

 

 

 

 

 

This entry was posted in Uncategorized. Bookmark the permalink.

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s