PhantomJS parser

JavaScript

var startUrl = 'https://www.dev2qa.com/phantomjs-example/';
var threadHost = 'https://www.dev2qa.com/phantomjs-example/';

var system = require('system');

var fs = require('fs');
var pagesPath = 'pages.txt';
//fs.touch(pagesPath);
var pages = fs.read(pagesPath).split('|');
console.log('read data from "' + pagesPath + '": ' + pages.length);

var threadsPath = 'threads.txt';
var threads = fs.read(threadsPath).split('|');
console.log('read data from "' + threadsPath + '": ' + threads.length);

var webpage = require('webpage');
var page = webpage.create();

page.onConsoleMessage = function(msg) {
  system.stderr.writeLine( msg );
};

function parser_init() {
	page.open(startUrl, function(status) {
		console.log('Page opened, status:' + status + '...');

		var someContent = page.evaluate(function() {
			console.log('Page evaluated...');
			//* get answer-links
			var linksAnswers = document.querySelectorAll("a.qtl");
			var linksAnswersHrefArray = [];
			console.log('Links @ page: ' + linksAnswers.length);
			for(var i = 0; i < linksAnswers.length; i++) {
				var href = linksAnswers[i].href.split('?').pop();
				if(threads.indexOf(href) == 1) {
					// thread not parsed yet
					var thread = webpage.create();

					thread.onConsoleMessage = function(msg) {
						system.stderr.writeLine( msg );
					};

					thread.open(threadHost + '?' + href, function(status) {
						console.log('thread opened, status:' + status);
					});
				}

				linksAnswersHrefArray.push(href);
				//var fileName = 'contents/file_' + i + '.txt';
				//fs.write(fileName, linksAnswersHrefArray[i], 'w');
				console.log(href);
			}
			return linksAnswersHrefArray;
		});
		console.log(someContent.join(',\n'));
		phantom.exit(0);
	}); 
}


parser_init();