学Python的应该都写过爬虫吧，如果希望提高爬虫的效率就要用到并发，可以选择的用多线程、多进程，还有最近很火的Gevent，据说是基于一种新的概念，协程，不管什么程，总之好用就行了。写一个爬虫有一系列的东西需要处理，如果有一个好用的框架就会事半功倍。 在42qu.com源码里面，有个教主写的爬虫框架，很好用，是基于Gevent的，处理url采用和web开发类似的映射方法，教主还写了一个简单的html处理库，extract，虽没有Beautifulsoup那样强大，但是简单好用，基本能满足常见的要求。HTTP请求使用了requests，requests是个处理HTTP的库，用官方的话说“Requests is an ISC Licensed HTTP library, written in Python, for human beings.”，for human beings，好吧，废话不说，上代码： 这里我将会把所需代码都贴出来，并贴一个示例。由于windows平台貌似没有Gevent支持，我还改写了一个多线程版本，一并放出。 首先是extract：
#coding:utf-8
import re
def extract(begin, end, html):
if not html:
return ''
start = html.find(begin)
if start >= 0:
start += len(begin)
if end is not None:
end = html.find(end, start)
if end is None or end >= 0:
return html[start:end].strip()
def extract_all(begin, end, html):
return map(str.strip, _extract_all(begin, end, html))
def _extract_all(begin, end, html):
if not html:
return ''
result = []
from_pos = 0
while True:
start = html.find(begin, from_pos)
if start >= 0:
start += len(begin)
endpos = html.find(end, start)
if endpos >= 0:
result.append(html[start:endpos])
from_pos = endpos+len(end)
continue
break
return result
def line_strip(txt):
if not txt:
return ''
txt = txt.replace(' ', ' ').split('\n')
return '\n'.join(i for i in [i.strip() for i in txt] if i)
def extract_strip(begin, end, html):
if not html:
return ''
t = extract(begin, end, html)
if t:
return strip_line(t)
def extract_map(begin, end, html, func):
txt = []
result = []
prepos = None
preend = 0
len_end = len(end)
len_begin = len(begin)
while True:
if prepos is None:
pos = html.find(begin)
else:
pos = html.find(begin, prepos)
if pos >= 0:
end = html.find(end, pos)
if pos < 0 or end < 0:
result.append(html[preend:])
break
end = end+len_end
result.append(html[preend:pos])
tmp = func(html[pos:end])
if tmp:
result.append(tmp)
prepos = pos+len_begin
preend = end
return ''.join(result)
if __name__ == '__main__':
pass
然后是爬虫框架，下面是原版
#coding:utf-8
import _env
from gevent.queue import Empty, Queue
import gevent
import gevent.monkey
import requests
from urlparse import urlparse, parse_qs
import re
gevent.monkey.patch_all()
class Bot(object):
cookie = None
headers = {}
def __init__(self, route):
self.queue = Queue()
self.route = route
def _fetch(self):
queue = self.queue
timeout = self.timeout
route = self.route
while True:
try:
url = queue.get(timeout=timeout+10)
except Empty:
return
headers = self.headers
if self.cookie:
headers['Cookie'] = self.cookie
req = requests.get(url, timeout=timeout, headers=headers, proxies=self.proxies)
p = urlparse(req.url)
cls, args = route.match(p.path)
if cls:
o = cls(req)
r = o.get(*args)
if r:
for i in r:
if i:
queue.put(i)
def run(self, num=10, timeout=60, proxies={}, cookie=None):
self.proxies = proxies
self.timeout = timeout
self.cookie = cookie
for i in xrange(num):
g = gevent.spawn(self._fetch)
g.join()
def put(self, url):
self.queue.put(url)
class Route(object):
def __init__(self):
self.map = []
def match(self, url):
for r, f in self.map:
m = r.match(url)
if m:
return f, m.groups()
return None, None
def __call__(self, path):
if not path.endswith('$'):
path += '$'
re_path = re.compile(path)
def _(func):
self.map.append((re_path, func))
return func
return _
route = Route()
bot = Bot(route)
from extract import extract, extract_all
class Page(object):
def __init__(self, req):
p = urlparse(req.url)
req.arguments = parse_qs(p.query, 1)
self.req = req
self.html = req.content
def get_argument(self, name, default=None):
result = self.req.arguments.get(name, None)
if result is None:
return default
return result[0].encode('utf-8', 'ignore')
def extract(self, begin, end):
return extract(begin, end, self.html)
def extract_all(self, begin, end):
return extract_all(begin, end, self.html)
然后是线程版的，适用于windows平台或者未安装Gevent的情况：
#coding:utf-8
import _env
import re
import time
import requests
from Queue import Queue
from threading import Thread
from urlparse import urlparse, parse_qs
ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.83 Safari/537.1'
class Bot(object):
cookie = None
headers = {}
def __init__(self, route):
self.queue = Queue()
self.route = route
def _fetch(self):
queue = self.queue
timeout = self.timeout
route = self.route
while True:
try:
url = queue.get(timeout=timeout+10)
except:
return
headers = self.headers
if self.cookie:
headers['Cookie'] = self.cookie
headers['User-Agent'] = ua
req = requests.get(url, timeout=timeout, headers=headers, proxies=self.proxies)
time.sleep(3)
p = urlparse(req.url)
cls, args = route.match(p.path)
if cls:
o = cls(req)
r = o.get(*args)
if r:
for i in r:
if i:
queue.put(i)
def run(self, num=6, timeout=10, proxies={}, cookie=cookie):
self.timeout = timeout
self.proxies = proxies
self.cookie = cookie
g = Thread()
g.setDaemon(1)
g.start()
for i in xrange(num):
g = Thread(target=self._fetch)
g.start()
#g.join()
def put(self, url):
self.queue.put(url)
class Route(object):
def __init__(self):
self.map = []
def match(self, url):
for r, f in self.map:
m = r.match(url)
if m:
return f, m.groups()
return None, None
def __call__(self, path):
if not path.endswith('$'):
path += '$'
re_path = re.compile(path)
def _(func):
self.map.append((re_path, func))
return func
return _
route = Route()
bot = Bot(route)
from extract import extract, extract_all
class Page(object):
def __init__(self, req):
p = urlparse(req.url)
req.arguments = parse_qs(p.query, 1)
self.req = req
self.html = req.content
def get_argument(self, name, default=None):
result = self.req.arguments.get(name, None)
if result is None:
return default
return result[0].encode('utf-8', 'ignore')
def extract(self, begin, end):
return extract(begin, end, self.html)
def extract_all(self, begin, end):
return extract_all(begin, end, self.html)
线程版和Gevent用法完全一样，下面是一个抓取豆瓣相册的例子：
#!/usr/bin/python
# -*- coding: utf-8 -*-
import _env
import _bot
import os
import urllib
from os.path import join, exists
from _bot import bot, route, Page
from extract import extract, extract_all
url_base = 'http://img1.douban.com/view/photo/raw/public/p%s.jpg'
url_tmp = 'http://movie.douban.com/subject/3395373/photos?type=S&start=%s&sortby=vote&size=a&subtype=a'
f_base = '/home/lerry/movie'
#f_base = u'E:\Download\photo\蝙蝠侠'
f_tmp = join(f_base, 'p%s.jpg')
@route('/subject/3395373/photos')
class url_list(Page):
def get(self):
page = self.get_argument('start')
page = int(page)
if page == 0:
for i in xrange(1, 41):
start = i*40
url = url_tmp % start
yield url
li = self.extract('class="poster-col4 clearfix"', '</ul>')
for i in extract_all('<li>', '</li>', li):
path = extract('a href="', '/">', i)
if not path:
continue
id = path[37:]
fpath = f_tmp % id
if exists(fpath):
continue
url = url_base % id
yield url
@route('/view/photo/raw/public/(.+)')
class single(Page):
def get(self, arg):
save_pic(self.html, arg)
def save_pic(content, fname):
fpath = join(f_base, fname)
f = open(fpath, 'wb')
f.write(content)
f.close()
print fname, 'saved'
if __name__ == '__main__':
bot.put('http://movie.douban.com/subject/3395373/photos?type=S&start=0&sortby=vote&size=a&subtype=a')
bot.run()