nodejs 爬虫笔记

var cheerio = require(‘cheerio‘);

// 通过 load 方法把 HTML 代码转换成一个 jQuery 对象
var $ = cheerio.load(‘
Hello world
‘);

// 可以使用与 jQuery 一样的语法来操作
$(‘h2.title‘).text(‘Hello there!‘);
$(‘h2‘).addClass(‘welcome‘);

console.log($.html());
// 将输出 
Hello there!

简单来说，cheerio就是服务器端的jQuery，去掉了jQuery的一些效果类和请求类等等功能后，仅保留核心对dom操作的部分，因此能够对dom进行和jQuery一样方便的操作。它是我们筛选数据的利器——把多余的html标签去掉，只留下我们想要的内容的重要工具。需要注意的是，cheerio 并不支持所有 jQuery 的查询语法，比如 $(‘a:first‘) 会报错 ，只能写成  $(‘a‘).first() ，在使用的时候需要注意。jQuery的相关选择器可以参考http://www.cnblogs.com/xiaxuexiaoab/p/7091527.html

npm install request cheerio mysql async -save

{
  "name": "crawler",
  "version": "1.0.0",
  "description": "crawler and mysql",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "async": "^2.5.0",
    "cheerio": "^1.0.0-rc.1",
    "mysql": "^2.13.0",
    "request": "^2.81.0"
  }
}

var request = require(‘request‘);
var cheerio = require(‘cheerio‘);


var url = ‘http://www.imooc.com/learn/857‘;

function videocrawler(url,callback){
       //获取页面
    request(url,function(err,res){
        if(err){
            callback(err);
        }
        
        var $ = cheerio.load(res.body.toString()); //利用cheerio对页面进行解析

        var videoList = [];
        
        $(‘.video li a‘).each(function(){
            var $title = $(this).parent().parent().parent().text().trim();
            var title = $title.split(‘\n‘);
            var text = $(this).text().trim();
            text = text.split(‘\n‘);
            //console.log(text);
            var time = text[1].match(/\((\d+\:\d+)\)/); 
            var item={
                title : title[0],
                url : ‘http://www.imooc.com‘+$(this).attr(‘href‘),
                name : text[0],
                duration : time[1]
            };
            var s = item.url.match(/video\/(\d+)/);
            //console.log(s);
            if(Array.isArray(s)){
                item.id = s[1];
                videoList.push(item);
            }
        });
            
        callback(null,videoList);
    });
}

videocrawler(url,function(err,videoList){
       if(err){
           return console.log(err);
       }
       console.log(videoList);
});

CREATE TABLE `video` (
  `id` int(11) NOT NULL,
  `title` varchar(255) NOT NULL,
  `url` varchar(255) NOT NULL,
  `name` varchar(255) NOT NULL,
  `duration` varchar(20) NOT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

var mysql = require(‘mysql‘);
var async = require(‘async‘);

var pool = mysql.createPool({
    host : ‘localhost‘,
    user : ‘root‘,
    password : ‘z2457098495924‘,
    database : ‘muke‘
});

pool.getConnection(function(err,connection){
    if(err){
        console.log(err);
    }
    connection.query(‘show tables‘,function(err,result){
         if(err){
             return console.log(‘show tables error:‘+err.message);
         }
         return console.log(result);
    });
    connection.release();
});

var mysql = require(‘mysql‘);
var async = require(‘async‘);

var pool = mysql.createPool({
    host : ‘localhost‘,
    user : ‘root‘,
    password : ‘z2457098495924‘,
    database : ‘muke‘
});

exports.videoSave = function(list,callback){
    
    pool.getConnection(function(err,connection){
        if(err){
            return callback(err);
        }
        var insertsql = ‘insert into video(id,title,url,name,duration) values(?,?,?,?,?)‘;
        async.eachSeries(list,function(item,next){
            connection.query(insertsql,[item.id,item.title,item.url,item.name,item.duration],next);
        },callback);
        connection.release();
    });
};

var request = require(‘request‘);
var cheerio = require(‘cheerio‘);

exports.videocrawler = function(url,callback){
    request(url,function(err,res){
        if(err){
            callback(err);
        }

        var $ = cheerio.load(res.body.toString());

        var videoList = [];
        
        $(‘.video li a‘).each(function(){
            var $title = $(this).parent().parent().parent().text().trim();
            var title = $title.split(‘\n‘);
            var text = $(this).text().trim();
            text = text.split(‘\n‘);
            //console.log(text);
            var time = text[1].match(/\((\d+\:\d+)\)/); 
            var item={
                title : title[0],
                url : ‘http://www.imooc.com‘+$(this).attr(‘href‘),
                name : text[0],
                duration : time[1]
            };
            var s = item.url.match(/video\/(\d+)/);
            //console.log(s);
            if(Array.isArray(s)){
                item.id = s[1];
                videoList.push(item);
            }
        });
            
        callback(null,videoList);
    });
}

var async = require(‘async‘);var video = require(‘./video‘);
var save = require(‘./save‘);
var url = ‘http://www.imooc.com/learn/857‘;
var videolist;

async.series([
        //获取视频信息
        function(done){
            video.videocrawler(url,function(err,list){
                videolist = list;
                done(err);
            });
        },
        //保存视频信息
        function(done){
            save.videoSave(videolist,done);
        },

    ],function(err){
    if(err){
        console.log(err);
    }
    console.log(‘完成‘);
    process.exit(0);
})