HEADER = {
'User-Agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36(KHTML, like Gecko)Chrome/93.0.4577.63 Safari/537.36",                                                         'Cookie':"xxxxxxx",
}

BASE_URL = "http://www.guishiyun.com" #賦值網(wǎng)站根域名作為全局變量，方便調(diào)用

res = requests.get(BASE_URL + 
                    "/res_list.aspx?rid=9&tags=1-21,12-96,2-24,3-70",
                    headers=HEADER).text #發(fā)起請(qǐng)求，獲得資源列表頁(yè)面的html

import re #導(dǎo)入re 正則表達(dá)式包

pattern = r'var zNodes = (\[\s*[\s\S]*\])'
#定義正則表達(dá)式，規(guī)則：找出以"var zNodes = [ \n"開(kāi)頭，含有"[多個(gè)字符或空格]"的字符，并且以"]"結(jié)尾的文本 （相關(guān)知識(shí)不熟悉的可以簡(jiǎn)單看看菜鳥(niǎo)的正則表達(dá)式）
result = re.findall(pattern, res, re.M | re.I)
#python正則表達(dá)式，查找res中符合pattern規(guī)則的文本。re.M多行匹配，re.I忽略大小寫。

def getRootText():
    res = requests.get(BASE_URL +
                       "/res_list.aspx?rid=9&tags=1-21,12-96,2-24,3-70",
                       headers=HEADER).text #請(qǐng)求
    pattern = r'var zNodes = (\[\s*[\s\S]*\])'
    result = re.findall(pattern, res, re.M | re.I)
    return result[0] #獲得篩選結(jié)果 [{id: 1322, pI': 1122, name: '九年級(jí)上冊(cè)', open: False, url: ?catId=1322&tags=1-21%2c12-96%2c2-24%2c3-70&rid=9#bottom_content', target: '_self'}, {...},{...}]

import demjson
def textToDict(text):
    data = demjson.decode(text)    
    #獲得篩選結(jié)果[{'id': 1322, 'pId': 1122, 'name': '九年級(jí)上冊(cè)', 'open': False, 'url': '?catId=1322&tags=1-21%2c12-96%2c2-24%2c3-70&rid=9#bottom_content', 'target': '_self'}, {...},{...}]
    return data

def getUrls(dictData):
    list = []
    pattern = r'第[\s\S]*?章' #正則規(guī)則：找出以"第"開(kāi)頭，中間包含多個(gè)空格和文字，以"章"結(jié)尾的文本
    for data in dictData:	#遍歷上文轉(zhuǎn)換得到的dict數(shù)組對(duì)象
        if len(re.findall(pattern, data['name'])) != 0:
            list.append(data['url']) #如果符合則將該url添加到列表中
    return list

def download(urlList): # urlList是上面獲得的list
	for url in urlList:
        res = requests.get(BASE_URL + '/res_list.aspx/' + url, HEADER).text #完整url請(qǐng)求，獲得頁(yè)面html

root = etree.HTML(res) # 構(gòu)造一個(gè)xpath對(duì)象
liList = root.xpath('//div[@class="res_list"]//ul//li') #xpath語(yǔ)法，返回多個(gè)<li>及子元素對(duì)象的列表

for li in liList:
	name = li.xpath('.//div[@class="info_area"]//div//h1//text()')
	name = name[0] # xpath返回的是包含name的列表，從中提取字符串 print(name): 1.1 反比例函數(shù)
    btnurl = li.xpath('.//div[@class="button_area"]//@onclick') # 獲得onlick內(nèi)的字符串 "window.open('res_view.aspx....')"
    pattern = r'\(\'([\s\S]*?)\'\)'# 只需要window.open內(nèi)的url，所以采用正則提取出來(lái)。
    btnurl1 = re.findall(pattern, btnurl[0])

 res1 = requests.get(BASE_URL + '/' + btnurl1[0], HEADER).text

VIEWSTATE = '__VIEWSTATE'              # 全局變量，定義屬性名稱
VIEWSTATEGENERATOR = '__VIEWSTATEGENERATOR'
EVENTVALIDATION = '__EVENTVALIDATION'
BUTTON = 'BUTTON'
BUTTON_value = '下 載 資 源'

root1 = etree.HTML(res1) # res1是之前代碼段請(qǐng)求的html文本
form = root1.xpath('//form[@id="form1"]') # xpath定位到form
action = root1.xpath('//form[@id="form1"]/@action') 
action = re.findall(r'(/[\S]*?&[\S]*?)&', action[0], re.I) #正則表達(dá)式獲取form中action函數(shù)里的url
VIEWSTATE_value = form[0].xpath(
                './/input[@name="__VIEWSTATE"]//@value') #獲取參數(shù)值
VIEWSTATEGENERATOR_value = form[0].xpath(
                './/input[@name="__VIEWSTATEGENERATOR"]//@value')#獲取參數(shù)值
EVENTVALIDATION_value = form[0].xpath(
                './/input[@name="__EVENTVALIDATION"]/@value')#獲取參數(shù)值
data = {	# post提交所需要的data參數(shù)									
          VIEWSTATE: VIEWSTATE_value,
          VIEWSTATEGENERATOR: VIEWSTATEGENERATOR_value,
          EVENTVALIDATION: EVENTVALIDATION_value,
          BUTTON: BUTTON_value
       }
res2 = requests.post(BASE_URL + action[0],data=data,headers=HEADER).text #發(fā)起請(qǐng)求

downUrl = re.search(r'\<script\>[\s]*?location\.href\s=\s\'([\S]*?)\'',res2,re.I) #正則篩選出url
 downUrl_text = downUrl.group(1)

downPPT = requests.get(BASE_URL+downUrl_text,headers=HEADER)
            with open(f'./test/{name}.ppt','wb') as f: #將下載的數(shù)據(jù)以二進(jìn)制的形式寫入到當(dāng)前項(xiàng)目下test文件夾中，并且做好命名。name參數(shù)在上文中已經(jīng)獲得。
                f.write(downPPT.content)

if(r.sadd(BASE_URL + action[0],'1')==0): # sadd是redis添加鍵值的方法，如果==0說(shuō)明已經(jīng)存在，添加失敗。
                continue

import re
import requests
from lxml import etree
import demjson
import redis

pool = redis.ConnectionPool(host='localhost', port=6379, decode_responses=True)
r = redis.Redis('localhost',6379,decode_responses=True)


BASE_URL = "http://www.guishiyun.com"
HEADER = {
    'User-Agent':
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
    'Cookie':
    "xxx",
}
VIEWSTATE = '__VIEWSTATE'
VIEWSTATEGENERATOR = '__VIEWSTATEGENERATOR'
EVENTVALIDATION = '__EVENTVALIDATION'
BUTTON = 'BUTTON'
BUTTON_value = '下 載 資 源'


def getRootText():
    res = requests.get(BASE_URL +
                       "/res_list.aspx?rid=9&tags=1-21,12-96,2-24,3-70",
                       headers=HEADER).text
    pattern = r'var zNodes = (\[\s*[\s\S]*\])'
    result = re.findall(pattern, res, re.M | re.I)
    return result[0]


def textToDict(text):
    data = demjson.decode(text)
    print(data)
    return data


def getUrls(dictData):
    list = []
    pattern = r'第[\s\S]*?章'
    for data in dictData:
        if len(re.findall(pattern, data['name'])) != 0:
            list.append(data['url'])
    return list


def download(urlList):
    global r
    for url in urlList:
        res = requests.get(BASE_URL + '/res_list.aspx/' + url, HEADER).text
        root = etree.HTML(res)
        liList = root.xpath('//div[@class="res_list"]//ul//li')
        for li in liList:
            name = li.xpath('.//div[@class="info_area"]//div//h1//text()')
            name = name[0]
            btnurl = li.xpath('.//div[@class="button_area"]//@onclick')
            pattern = r'\(\'([\s\S]*?)\'\)'
            btnurl1 = re.findall(pattern, btnurl[0])
            res1 = requests.get(BASE_URL + '/' + btnurl1[0], HEADER).text
            root1 = etree.HTML(res1)
            form = root1.xpath('//form[@id="form1"]')
            action = root1.xpath('//form[@id="form1"]/@action')
            action = re.findall(r'(/[\S]*?&[\S]*?)&', action[0], re.I)
            VIEWSTATE_value = form[0].xpath(
                './/input[@name="__VIEWSTATE"]//@value')
            VIEWSTATEGENERATOR_value = form[0].xpath(
                './/input[@name="__VIEWSTATEGENERATOR"]//@value')
            EVENTVALIDATION_value = form[0].xpath(
                './/input[@name="__EVENTVALIDATION"]/@value')
            data = {
                VIEWSTATE: VIEWSTATE_value,
                VIEWSTATEGENERATOR: VIEWSTATEGENERATOR_value,
                EVENTVALIDATION: EVENTVALIDATION_value,
                BUTTON: BUTTON_value
            }
            if(r.sadd(BASE_URL + action[0],'1')==0):
                continue
            res2 = requests.post(BASE_URL + action[0],data=data,headers=HEADER).text
            downUrl = re.search(r'\<script\>[\s]*?location\.href\s=\s\'([\S]*?)\'',res2,re.I)
            downUrl_text = downUrl.group(1)
            if(r.sadd(BASE_URL+downUrl_text,BASE_URL+downUrl_text,downUrl_text)==0):
                continue
            downPPT = requests.get(BASE_URL+downUrl_text,headers=HEADER)
            with open(f'./test/{name}.ppt','wb') as f:
                f.write(downPPT.content)

def main():
    text = getRootText()
    dictData = textToDict(text)
    list = getUrls(dictData)
    # download(list)


if __name__ == '__main__':
    main()