I am using Node v0.10.11 on Ubuntu 12.04. I can’t understand what I’m missing is that the URL streams work with the request module. This program tries to go to the mailing list site, find download links for each month, and then download pages for each month.
mikeal readme says: "The first argument can be either a URL or an option object. The only required parameter is uri, all the rest are optional. uri || url - fully qualified uri or parsed url object from url.parse ()"
If I call url.parse (www.targeturl.com), I get [Error: options.uri - required] If I do not use url.parse, I get [Error: Invalid URI "www.freelists.org/archive/ si-list / 06-2013 "] (this link works fine in my browsers)
I reduced the code to 42 lines. Any advice is welcome
var request = require('request'), url = require('url'), stream = require('stream'), cheerio = require('cheerio'), // a reduced jQuery style DOM library Transform = require('stream').Transform var DomStripStream = function(target) { this.target = target; stream.Transform.call(this,{objectMode: true}); } DomStripStream.prototype = Object.create( Transform.prototype, {constructor: {value: DomStripStream}} ) DomStripStream.prototype.write = function () { this._transform.apply(this, arguments); }; DomStripStream.prototype.end = function () { this._transform.apply(this, arguments); this.emit("end"); }; DomStripStream.prototype._transform = function(chunk, encoding, callback) { chunk = chunk ? chunk.toString() : ""; $ = cheerio.load(chunk); domLinks = $(this.target); $(domLinks).each(function (i, link) { currLink = 'www.freelists.org' + $(link).attr('href') // currLink = url.parse(currLink) request(currLink, function (error, response, body) { console.log(error); }) }); } var fs = require("fs"), output = fs.createWriteStream("out.txt"), mainPage = new DomStripStream('td a') request('http://www.freelists.org/archive/si-list'). pipe(mainPage). pipe(output);
source share