var keyword = "d3.js";

//@input(keyword, 查询关键字, 爬取该关键字搜索出来的京东商品) 

var comment_count = 100;

//@input(comment_count, 爬取的评论数, 最多爬取多少条评论) 

var page_count = comment_count / 10; 

keyword = keyword.trim(); 

var scanUrls = []; 

scanUrls.push("http://search.jd.com/Search?keyword=" + keyword.replace(/ /g, "+") + "&enc=utf-8&scrolling=y&page=200"); 

var helperUrlRegexes = []; 

helperUrlRegexes.push("http://search\\.jd\\.com/Search\\?keyword=" + keyword.replace(/ /g, "\\+").replace(/\./g, "\\.") + "&enc=utf-8&scrolling=y&page=\\d+"); 

var configs = { 

domains: ["search.jd.com", "item.jd.com", "club.jd.com"], scanUrls: scanUrls, contentUrlRegexes: ["http://item\\.jd\\.com/\\d+.html"], helperUrlRegexes: helperUrlRegexes, interval: 10000, fields: [ { // 第一个抽取项 name: "title", selector: "//div[@id='name']/h1", required: true }, { // 第一个抽取项 name: "productid", selector: "//div[contains(@class,'fl')]/span[2]", required: true }, { name: "comments", selector: "//div[@id='comment-pages']/span", repeated: true, children: [ { name: "page", selector: "//text()" }, { name: "comments", sourceType: SourceType.AttachedUrl, attachedUrl: "http://club.jd.com/productpage/p-{$.productid}-s-0-t-3-p-{page}.html", selectorType: SelectorType.JsonPath, selector: "$.comments", repeated: true, children:[ { name: "com_content", selectorType: SelectorType.JsonPath, selector: "$.content" }, { name: "com_nickname", selectorType: SelectorType.JsonPath, selector: "$.nickname" } ] 


} ] 

} ] 

};

 configs.afterDownloadPage = function(page, site) { var matches = /item\.jd\.com\/(\d+)\.html/.exec(page.url); 

if (!matches) return page; var commentUrl = "http://club.jd.com/productpage/p-"+matches[1]+"-s-0-t-3-p-0.html"; 

var result = site.requestUrl(commentUrl); 

var data = JSON.parse(result); var commentCount = data.productCommentSummary.commentCount; 

var pages = commentCount / 10; 

if (pages > page_count) pages = page_count; var pageHtml = "<div id=\"comment-pages\">"; 

for (var i = 0; i < pages; i++) { pageHtml += "<span>" + i + "</span>"; } pageHtml += "</div>"; 

var index = page.raw.indexOf("</body>"); 

page.raw = page.raw.substring(0, index) + pageHtml + page.raw.substring(index); return page; }; 

var dataSku = 0; configs.onProcessHelperPage = function(page, content, site) { var num = parseInt(extract(content, "//*[@id='J_goodsList']/ul/li[1]/@data-sku")); 

if (dataSku === 0) { 

dataSku = isNaN(num) ? 0 : num; } 

else if (dataSku === num) {

 dataSku = 0; return false; } var currentPageNum = parseInt(page.url.substring(page.url.indexOf("&page=") + 6)); 

if (currentPageNum === 0) { 

currentPageNum = 1; 


} var pageNum = currentPageNum + 1; var nextUrl = page.url.replace("&page=" + currentPageNum, "&page=" + pageNum); 

site.addUrl(nextUrl); return true; }; 

configs.afterExtractPage = function(page, data) { 

if (data.comments === null || data.comments === undefined) 

return data; var comments = []; 

for (var i = 0; i < data.comments.length; i++) { 

var p = data.comments[i]; for (var j = 0; j < p.comments.length; j++) { comments.push(p.comments[j]); 

} 

} 

data.comments = comments; 
return data; 

};

 var crawler = new Crawler(configs); crawler.start();

//牛妹双11快乐，大家双11快乐(*^▽^*)