# -*- coding:utf-8 -*-

#IO密集型任務(wù)
#多個(gè)進(jìn)程同時(shí)下載多個(gè)網(wǎng)頁(yè)
#利用Queue+多進(jìn)程
#由于是IO密集型,所以同樣可以利用threading模塊

import multiprocessing

def main():
  tasks = multiprocessing.JoinableQueue()
  results = multiprocessing.Queue()
  cpu_count = multiprocessing.cpu_count() #進(jìn)程數(shù)目==CPU核數(shù)目

  create_process(tasks, results, cpu_count)  #主進(jìn)程馬上創(chuàng)建一系列進(jìn)程,但是由于阻塞隊(duì)列tasks開(kāi)始為空,副進(jìn)程全部被阻塞
  add_tasks(tasks) #開(kāi)始往tasks中添加任務(wù)
  parse(tasks, results) #最后主進(jìn)程等待其他線程處理完成結(jié)果


def create_process(tasks, results, cpu_count):
  for _ in range(cpu_count):
    p = multiprocessing.Process(target=_worker, args=(tasks, results)) #根據(jù)_worker創(chuàng)建對(duì)應(yīng)的進(jìn)程
    p.daemon = True #讓所有進(jìn)程可以隨主進(jìn)程結(jié)束而結(jié)束
    p.start() #啟動(dòng)

def _worker(tasks, results):
  while True:  #因?yàn)榍懊嫠芯€程都設(shè)置了daemon=True,故不會(huì)無(wú)限循環(huán)
    try:
      task = tasks.get()  #如果tasks中沒(méi)有任務(wù),則阻塞
      result = _download(task)
      results.put(result)  #some exceptions do not handled
    finally:
      tasks.task_done()

def add_tasks(tasks):
  for url in get_urls(): #get_urls() return a urls_list
    tasks.put(url)

def parse(tasks, results):
  try: 
    tasks.join()
  except KeyboardInterrupt as err:
    print "Tasks has been stopped!"
    print err

  while not results.empty():
    _parse(results)



if __name__ == '__main__':
  main()

def handler():
  futures = set()

  with concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count) as executor:
    for task in get_task(tasks):
      future = executor.submit(task)
      futures.add(future)

def wait_for(futures):
  try:
    for future in concurrent.futures.as_completed(futures):
      err = futures.exception()
      if not err:
        result = future.result()
      else:
        raise err
  except KeyboardInterrupt as e:
    for future in futures:
      future.cancel()
    print "Task has been canceled!"
    print e
  return result