from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.text import TextResponse
from selenium.webdriver import ActionChains
 
class TaobaoMiddleware(object):
 
 #處理請(qǐng)求函數(shù)
 def process_request(self,request,spider):
  #聲明一個(gè)Options對(duì)象
  opt = Options()
  #給對(duì)象添加一個(gè)--headless參數(shù),表示無頭啟動(dòng)
  opt.add_argument('--headless')
  #把配置參數(shù)應(yīng)用到驅(qū)動(dòng)創(chuàng)建的對(duì)象
  driver = webdriver.Chrome(options=opt)
  #打開requests中的地址
  driver.get(request.url)
 
  #讓瀏覽器滾動(dòng)到底部
  for x in range(1,11):
   j = x / 10
   js = "document.documentElement.scrollTop = document.documentElement.scrollHeight*%f"%j
   driver.execute_script(js)
   #每次滾動(dòng)等待0.5s
   time.sleep(5)
 
  #獲取下一頁按鈕的標(biāo)簽
  next_btn =driver.find_element_by_xpath('//span[contains(text(),"下一頁")]')
  #睡眠0.5秒
  time.sleep(0.5)
  #對(duì)下一頁標(biāo)簽進(jìn)行鼠標(biāo)右鍵觸發(fā)事件
  ActionChains(driver).context_click(next_btn).click().perform()
  # driver.save_screenshot('截圖.png')
  #把驅(qū)動(dòng)對(duì)象獲得的源碼賦值給新變量
  page_source = driver.page_source
  #退出
  driver.quit()
 
  #根據(jù)網(wǎng)頁源代碼,創(chuàng)建Htmlresponse對(duì)象
  response = HtmlResponse(url=request.url,body=page_source,encoding='utf-8',request=request)
  #因?yàn)榉祷氐氖俏谋鞠?所以需要指定字符編碼格式
 
  return response
 
 def process_response(self,request,response,spider):
 
  return response
 
 def process_exception(self,request,exception,spider):
  pass