#!/usr/bin/env python
#coding: utf-8
import urllib
import urllib2
import os
import re
import sys
#顯示下載進度
def schedule(a,b,c):
  '''''
  a:已經(jīng)下載的數(shù)據(jù)塊
  b:數(shù)據(jù)塊的大小
  c:遠程文件的大小
  '''
  per = 100.0 * a * b / c
  if per > 100 :
    per = 100
  print '%.2f%%' % per
#獲取html源碼
def getHtml(url):
  page = urllib.urlopen(url)
  html = page.read()
  return html
#下載圖片
def downloadImg(html, num, foldername):
  picpath = '%s' % (foldername) #下載到的本地目錄
  if not os.path.exists(picpath): #路徑不存在時創(chuàng)建一個
    os.makedirs(picpath)
  target = picpath+'/%s.jpg' % num
  myItems = re.findall('<p><a href="http:\/\/www.mzitu.com/.*?" ><img src="(.*?)" alt=".*?" /></a></p>',html,re.S)
  print 'Downloading image to location: ' + target
  urllib.urlretrieve(myItems[0], target, schedule)
#正則匹配分頁
def findPage(html):
  myItems = re.findall('<span>(\d*)</span>', html, re.S)
  return myItems.pop()
#正則匹配列表
def findList(html):
  myItems = re.findall('<h2><a  title="(.*?)" target="_blank">.*?</a></h2>', html, re.S)
  return myItems
#總下載
def totalDownload(modelUrl):
  listHtml5 = getHtml(modelUrl)
  listContent = findList(listHtml)
  for list in listContent:
    html = getHtml('http://www.mzitu.com/' + str(list[0]))
    totalNum = findPage(html)
    for num in range(1, int(totalNum)+1):
      if num == 1:
        url = 'http://www.mzitu.com/' + str(list[0])
        html5 = getHtml(url)
        downloadImg(html5, str(num), str(list[1]))
      else:
        url = 'http://www.mzitu.com/' + str(list[0]) + '/'+str(num)
        html5 = getHtml(url)
        downloadImg(html5, str(num), str(list[1]))
if __name__ == '__main__':
  listHtml = getHtml('http://www.mzitu.com/model')
  #這是其中一個模塊的url，可以添加不同的模塊url從而達到整站爬取。
  for model in range(1, int(findPage(listHtml))+1):
    if model == 1:
      modelUrl = 'http://www.mzitu.com/model'
      totalDownload(modelUrl)
    else:
      modelUrl = 'http://www.mzitu.com/model/page/' + str(model)
      totalDownload(modelUrl)
  print "Download has finished."