文件結(jié)構(gòu)
html_downloader.py - 下載網(wǎng)頁(yè)html內(nèi)容
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import urllib2
class HtmlDownloader(object):
def downlod(self, url):
if url is None:
return None
response = urllib2.urlopen(url)
if response.getcode() != 200:
return None
return response.read()
html_outputer.py - 輸出結(jié)果到文件中
#!/usr/bin/python
# -*- coding: UTF-8 -*-
class HtmlOutputer(object):
def collect_data(self, movie_data):
if movie_data is None:
return
fout = open('output.html', 'a+')
for data in movie_data:
print data['name'] + '|', data['rate'] + '|', data['actor'], '\n'
fout.write('%s,' % data['name'].encode('utf-8'))
fout.write('%s,' % data['rate'])
fout.write('%s\n' % data['actor'].encode('utf-8'))
fout.close()
html_parser.py: 解析器：解析html的dom樹(shù)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
class HtmlParser(object):
def __init__(self):
pass
def parser_html(self, cnt):
if cnt is None:
return
soup = BeautifulSoup(cnt, 'html.parser', from_encoding='utf-8')
# movie_name, movie_desc, movie_rate =
return self.get_movie_names(soup)
def get_movie_names(self, soup):
movie_data = []
movie_all = soup.find('div', class_='article').find_next('table').find_next_sibling('div').find_next_sibling('div').find_all('table')
count = 1
for movie_one in movie_all:
movie_data.append(self.get_movie_name(movie_one))
# if count > 2:
# break
count += 1
return movie_data
def get_movie_name(self, cnt):
info = {}
soup = BeautifulSoup(str(cnt), 'html.parser', from_encoding='utf-8')
movie_one = soup.find('tr', class_='item').find_next('td').find_next_sibling('td').find('div', class_='pl2')
info['name'] = movie_one.find('a').get_text().replace("\n", "").replace(" ", "")
info['actor'] = movie_one.find('p', class_='pl').get_text().replace("\n", "").replace(" ", "")
info['rate'] = movie_one.find('div', class_='star clearfix').find('span', class_='rating_nums').get_text()
return info
spider_main.py - 主函數(shù)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import html_parser, html_outputer, html_downloader
class SpiderMain(object):
def __init__(self):
self.parser = html_parser.HtmlParser()
self.outputer = html_outputer.HtmlOutputer()
self.downloader = html_downloader.HtmlDownloader()
def craw(self, url):
html_cnt = self.downloader.downlod(url)
movie_data = self.parser.parser_html(html_cnt)
self.outputer.collect_data(movie_data)
if __name__ == '__main__':
url = 'https://movie.douban.com/tag/2017?start=100&type=T'
spider = SpiderMain()
spider.craw(url)
綜述
其實(shí)就是使用了urllib2和BeautifulSoup庫(kù)，沒(méi)啥好說(shuō)的，你也可以直接改url，然后更改html_parser.py文件來(lái)滿足你自己的爬蟲(chóng)需求。當(dāng)前也可以更改html_outputer.py來(lái)定義保存格式，目前是csv。
更多文章、技術(shù)交流、商務(wù)合作、聯(lián)系博主
微信掃碼或搜索：z360901061
微信掃一掃加我為好友
QQ號(hào)聯(lián)系： 360901061
您的支持是博主寫(xiě)作最大的動(dòng)力，如果您喜歡我的文章，感覺(jué)我的文章對(duì)您有幫助，請(qǐng)用微信掃描下面二維碼支持博主2元、5元、10元、20元等您想捐的金額吧，狠狠點(diǎn)擊下面給點(diǎn)支持吧，站長(zhǎng)非常感激您！手機(jī)微信長(zhǎng)按不能支付解決辦法：請(qǐng)將微信支付二維碼保存到相冊(cè)，切換到微信，然后點(diǎn)擊微信右上角掃一掃功能，選擇支付二維碼完成支付。
【本文對(duì)您有幫助就好】元
