# coding: utf-8
new_urls = set()
data = {}
class SpiderMan(object):
 def __init__(self):
  #調(diào)度器內(nèi)包含其它四個元件，在初始化調(diào)度器的時候也要建立四個元件對象的實例
  self.manager = UrlManager()
  self.downloader = HtmlDownloader()
  self.parser = HtmlParser()
  self.output = DataOutput()
 def spider(self, origin_url):
  #添加初始url
  self.manager.add_new_url(origin_url)
  #下面進入主循環(huán)，暫定爬取頁面總數(shù)小于100
  num = 0
  while(self.manager.has_new_url() and self.manager.old_url_size()<100):
   try:
    num = num + 1
    print "正在處理第{}個鏈接".format(num)
    #從新url倉庫中獲取url
    new_url = self.manager.get_new_url()
    #調(diào)用html下載器下載頁面
    html = self.downloader.download(new_url)
    #調(diào)用解析器解析頁面，返回新的url和data
    try:
     new_urls, data = self.parser.parser(new_url, html)
    except Exception, e:
     print e
    for url in new_urls:
     self.manager.add_new_url(url)
    #將已經(jīng)爬取過的這個url添加至老url倉庫中
    self.manager.add_old_url(new_url)
    #將返回的數(shù)據(jù)存儲至文件
     self.output.store_data(data)
     print "store data succefully"
    print "第{}個鏈接已經(jīng)抓取完成".format(self.manager.old_url_size())
   except Exception, e:
    print e
  #爬取循環(huán)結(jié)束的時候?qū)⒋鎯Φ臄?shù)據(jù)輸出至文件
  self.output.output_html()