本篇文章主要介绍了"极客学院课程爬虫Requests"，主要涉及到方面的内容，对于其他编程jrs看球网直播吧_低调看直播体育app软件下载_低调看体育直播感兴趣的同学可以参考一下：
1、Requests抓取网页2、re.sub换页3、正则表达式匹配内容# -*- coding: utf-8 -*-
import requests
impor...
1、Requests抓取网页
2、re.sub换页
3、正则表达式匹配内容
# -*- coding: utf-8 -*-
import requests
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
# 在windows下面命令提示符默认编码是GBK 要爬内容为UTF-8
class spider(object):
def changepage(self, url, total_page):
now_page = int(re.search('pageNum=(\d+)',url,re.S).group(1))
page_group = []
for i in range(now_page,total_page+1):
link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
page_group.append(link)
return page_group
def getsource(self,url):
html = requests.get(url)
return html.text
def geteveryclass(self,source):
everyclass = re.findall('<>',source,re.S)
return everyclass
def getinfo(self,eachclass):
info = {}
info['title'] = re.search('(.*?)',eachclass,re.S).group(2)
info['des'] = re.search('(.*?)',eachclass,re.S).group(2)
timeandlevel = re.findall('(.*?)',eachclass,re.S)
info['classtime'] = timeandlevel[0]
info['classlevel'] = timeandlevel[1]
return info
def saveinfo(self,classinfo):
f = open('info.txt','a')
for each in classinfo:
f.writelines('title: ' + each['title'] + '\n')
f.writelines('des: ' + each['des'] + '\n')
f.writelines('time: ' + each['classtime'] + '\n')
f.writelines('level: ' + each['classlevel'] + '\n')
f.writelines('\n\n')
f.close()
if __name__ == '__main__':
classinfo=[]
url = 'http://www.jikexueyuan.com/course/?pageNum=1'
jikespider = spider() # 类的实例 jikespider
all_links = jikespider.changepage(url,3) # 得到所有的链接
# print all_links # 测试
for link in all_links:
print '正在处理页面：' + link
html = jikespider.getsource(link)
everyclass = jikespider.geteveryclass(html)
# print everyclass # 得到网页内容测试
for each in everyclass:
info = jikespider.getinfo(each)
classinfo.append(info)
jikespider.saveinfo(classinfo)
以上就介绍了极客学院课程爬虫Requests，包括了方面的内容，希望对其他编程jrs看球网直播吧_低调看直播体育app软件下载_低调看体育直播有兴趣的朋友有所帮助。
本文网址链接：http://www.codes51.com/article/detail_340389.html
